From f8b33a65b89a17f0d4855310b35f510dd457103a Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Tue, 24 Jan 2023 14:15:17 -0800 Subject: [PATCH] Identity averages are now working! Signed-off-by: James Ketrenos --- client/package-lock.json | 9 + client/package.json | 1 + client/src/App.css | 35 +++- client/src/App.tsx | 213 ++++++++++++++++++--- server/db/photos.js | 1 + server/routes/identities.js | 369 ++++++++++++++++++++++++------------ 6 files changed, 483 insertions(+), 145 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index ae8fc59..f0949c9 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -15,6 +15,7 @@ "@types/node": "^18.11.18", "@types/react": "^18.0.26", "@types/react-dom": "^18.0.10", + "moment": "^2.29.4", "react": "^18.2.0", "react-dom": "^18.2.0", "react-resizable-panels": "^0.0.34", @@ -14346,6 +14347,14 @@ "mkdirp": "bin/cmd.js" } }, + "node_modules/moment": { + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==", + "engines": { + "node": "*" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", diff --git a/client/package.json b/client/package.json index 83fb2c1..cc51be0 100644 --- a/client/package.json +++ b/client/package.json @@ -11,6 +11,7 @@ "@types/node": "^18.11.18", "@types/react": "^18.0.26", "@types/react-dom": "^18.0.10", + "moment": "^2.29.4", "react": "^18.2.0", "react-dom": "^18.2.0", "react-resizable-panels": "^0.0.34", diff --git a/client/src/App.css b/client/src/App.css index bb85f6c..4bb3688 100644 --- a/client/src/App.css +++ b/client/src/App.css @@ -35,6 +35,15 @@ div { flex-grow: 1; } +.Actions { + display: flex; + flex-direction: row; + justify-content: flex-end; + gap: 0.25rem; + padding: 0.25rem; + width: 100%; +} + .Identities { display: grid; user-select: none; @@ -64,27 +73,45 @@ div { flex-direction: column; } +.PhotoPanel { + display: flex; + flex-direction: column; + height: 100%; + width: 100%; + justify-content: center; +} + .Image .FaceBox { border: 1px solid red; -/* border-radius: 0.25rem;*/ position: absolute; } .Image .FaceBox:hover { background-color: rgba(255, 255, 255, 0.2); - box-shadow: 0px 0px 5px black; + box-shadow: 0px 0px 10px black; } .Image { display: flex; - width: 100%; - height: 100%; position: relative; background-size: contain !important; background-repeat: no-repeat no-repeat !important; background-position: 50% 50% !important; } +.PhotoPanel .FaceInfo { + padding: 0.25rem; + background-color: #444; + color: white; + margin-top: 0.25rem;} + +.PhotoPanel .ImageInfo { + padding: 0.25rem; + background-color: #222; + color: white; + margin-top: 0.25rem; +} + .UnknownFace { display: flex; align-items: center; diff --git a/client/src/App.tsx b/client/src/App.tsx index 7b91ef2..b847cb0 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -8,24 +8,32 @@ import { Routes, useParams } from "react-router-dom"; +import moment from 'moment'; import equal from "fast-deep-equal"; import './App.css'; const base = process.env.PUBLIC_URL; /* /identities -- set in .env */ -const makeFaceBoxes = (photo: any, dimensions: any, onFaceClick: any): any => { +const makeFaceBoxes = (photo: any, + dimensions: any, + onFaceClick: any, + onFaceEnter: any, + onFaceLeave: any, + ): any => { const faces: FaceData[] = photo.faces; let width: number, height: number, offsetLeft = 0, offsetTop = 0; /* If photo is wider than viewport, it will be 100% width and < 100% height */ if (photo.width / photo.height > dimensions.width / dimensions.height) { + console.log('Landscape'); width = dimensions.width; height = dimensions.height * photo.height / photo.width * dimensions.width / dimensions.height; offsetLeft = 0; offsetTop = (dimensions.height - height) * 0.5; } else { + console.log('Portrait'); width = dimensions.width * photo.width / photo.height * dimensions.height / dimensions.width; height = dimensions.height; @@ -48,8 +56,8 @@ const makeFaceBoxes = (photo: any, dimensions: any, onFaceClick: any): any => { height: Math.floor((face.bottom - face.top) * height) + "px" }} onClick={(e) => { onFaceClick(e, face) }} - onMouseEnter={(e) => { onFaceMouseEnter(e, face) }} - onMouseLeave={(e) => { onFaceMouseLeave(e, face) }} + onMouseEnter={(e) => { onFaceEnter(e, face) }} + onMouseLeave={(e) => { onFaceLeave(e, face) }} /> ) }); @@ -57,14 +65,26 @@ const makeFaceBoxes = (photo: any, dimensions: any, onFaceClick: any): any => { const Photo = ({ photoId, onFaceClick }: any) => { const [image, setImage] = useState(undefined); + const [faceInfo, setFaceInfo] = useState(''); const ref = useRef(null); const [dimensions, setDimensions] = React.useState({width: 0, height: 0}); + const onFaceEnter = (e: any, face: FaceData) => { + onFaceMouseEnter(e, face); + setFaceInfo(face.identity ? face.identity.displayName : 'Unknown'); + } + + const onFaceLeave = (e: any, face: FaceData) => { + setFaceInfo(''); + onFaceMouseLeave(e, face); + } + const faces = useMemo(() => { if (image === undefined || dimensions.height === 0) { return <>; } - return makeFaceBoxes(image, dimensions, onFaceClick); + return makeFaceBoxes(image, dimensions, + onFaceClick, onFaceEnter, onFaceLeave); }, [image, dimensions, onFaceClick]); const checkResize = useCallback(() => { @@ -78,7 +98,7 @@ const Photo = ({ photoId, onFaceClick }: any) => { setDimensions({ height: el.clientHeight, width: el.clientWidth - }) + }); } }, [setDimensions, dimensions]); @@ -106,15 +126,25 @@ const Photo = ({ photoId, onFaceClick }: any) => { return <> } - return (
- - { faces } + return (
+
+ + { faces } +
+
{ + moment(image.taken) + .format('MMMM Do YYYY, h:mm:ss a') + }, { + moment(image.taken) + .fromNow() + }.
+
{ faceInfo ? faceInfo : 'Hover over face for information.'}
); }; @@ -132,7 +162,10 @@ const onFaceMouseEnter = (e: any, face: FaceData) => { els.forEach(el => { el.classList.add('Active'); - }) + }); + + e.stopPropagation(); + e.preventDefault(); }; const onFaceMouseLeave = (e: any, face: FaceData) => { @@ -145,7 +178,10 @@ const onFaceMouseLeave = (e: any, face: FaceData) => { els.forEach(el => { el.classList.remove('Active'); - }) + }); + + e.stopPropagation(); + e.preventDefault(); }; const Face = ({ face, onFaceClick, title, ...rest }: any) => { @@ -180,10 +216,15 @@ type ClusterProps = { identity: IdentityData, setImage(image: number): void, setSelected(selected: number[]): void, - setIdentity(identity: IdentityData): void + setIdentity(identity: IdentityData | undefined): void + identities: IdentityData[], + setIdentities(identiteis: IdentityData[]): void }; -const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps) => { +const Cluster = ({ + identity, setIdentity, + identities, setIdentities, + setImage, setSelected }: ClusterProps) => { const relatedFacesJSX = useMemo(() => { const faceClicked = async (e: any, face: FaceData) => { if (!identity) { @@ -218,7 +259,17 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps) if (identity === undefined) { return <>; } - return identity.relatedFaces.map((face: FaceData) => { + + return identity.relatedFaces.map((face: FaceData, i: number) => { + if (i >= 1000) { + if (i === 1000) { + return
+ too many faces (${identity.relatedFaces.length - 1000} remaining) +
; + } else { + return; + } + } return (
{ + try { + const res = await window.fetch( + `${base}/api/v1/identities/${identity.identityId}`, { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' } + }); + const updated = await res.json(); + const index = identities + .findIndex((item: IdentityData) => + item.identityId === identity.identityId); + if (index !== -1) { + identities.splice(index, 1); + } + setIdentity(undefined); + setIdentities([...identities]); + } catch (error) { + console.error(error); + } + }; + const updateIdentity = async () => { try { const validFields = [ @@ -289,6 +361,7 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps) }); const created = await res.json(); setIdentity(created); + setIdentities([identity, ...identities]); } catch (error) { console.error(error); } @@ -321,8 +394,11 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps) value={identity.displayName} onChange={displayNameChanged} /> +
+ +
Faces: {identity.relatedFaces.length}
@@ -356,7 +432,9 @@ type IdentityData = { descriptors: number[], identityId: number displayName: string, - relatedFaces: FaceData[] + relatedFaces: FaceData[], + facesCount: number, + faceId: number }; interface IdentitiesProps { @@ -379,7 +457,7 @@ const Identities = ({ identities, onFaceClick } : IdentitiesProps) => { + title={`${identity.displayName} (${identity.facesCount})`}/>
); }); @@ -406,7 +484,8 @@ const App = () => { const [selectedIdentities, setSelectedIdentities] = useState([]); const [identity, setIdentity] = useState(undefined); const [image, setImage] = useState(0); - const { loading, data } = useApi( + const [guess, setGuess] = useState(undefined); + const { loading, data } = useApi( /* TODO: Switch away from using useApi */ `${base}/api/v1/identities` ); const [selected, setSelected] = useState([]); @@ -467,6 +546,9 @@ const App = () => { face.identity = identity; }); }); + data.sort((A: IdentityData, B: IdentityData) => { + return A.displayName.localeCompare(B.displayName); + }); setIdentities(data as IdentityData[]); } }, [data]); @@ -485,6 +567,42 @@ const App = () => { } } + const mergeIdentity = async () => { + if (selectedIdentities.length === 0) { + window.alert('You need to select an identity first (CTRL+CLICK)'); + return; + } + if (!identity) { + return; + } + try { + let res = await window.fetch( + `${base}/api/v1/identities/faces/add/${selectedIdentities[0]}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ faces: identity.relatedFaces + .map(face => face.faceId) }) + }); + await res.json(); + res = await window.fetch( + `${base}/api/v1/identities/${identity.identityId}`, { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' } + }); + const updated = await res.json(); + const index = identities + .findIndex((item: IdentityData) => + item.identityId === identity.identityId); + if (index !== -1) { + identities.splice(index, 1); + } + setIdentity(undefined); + setIdentities([...identities]); + } catch (error) { + console.error(error); + } + }; + const markSelectedIncorrectIdentity = async () => { if (!identity) { return; @@ -505,7 +623,6 @@ const App = () => { }; const changeSelectedIdentity = async () => { - if (selectedIdentities.length === 0) { window.alert('You need to select an identity first (CTRL+CLICK)'); return; @@ -525,6 +642,21 @@ const App = () => { } }; + const guessIdentity = async () => { + try { + const res = await window.fetch( + `${base}/api/v1/identities/faces/guess/${selected[0]}`, { + method: 'GET', + headers: { 'Content-Type': 'application/json' }, + }); + const faces = await res.json(); + console.log(faces); + setGuess(faces[0]); + } catch (error) { + console.error(error); + } + }; + const markSelectedNotFace = async () => { try { const res = await window.fetch( @@ -537,12 +669,23 @@ const App = () => { }) }); const data = await res.json(); - removeFacesFromIdentities(data); + removeFacesFromIdentities(selected); } catch (error) { console.error(error); } }; + const deselectAll = () => { + const cluster = document.querySelector('.Cluster'); + if (!cluster) { + return; + } + [...cluster.querySelectorAll('.Selected')].forEach(item => { + item.classList.remove('Selected') + }); + setSelected([]); + }; + const onFaceClick = (e: any, face: FaceData) => { const identityId = face.identityId; const faceId = face.faceId; @@ -555,6 +698,9 @@ const App = () => { }); }; + const guessOnFaceClick = (e: any, face: FaceData) => { + }; + const identitiesOnFaceClick = (e: any, face: FaceData) => { const identitiesEl = document.querySelector('.Identities'); if (!identitiesEl) { @@ -600,6 +746,8 @@ const App = () => { } @@ -607,10 +755,17 @@ const App = () => { Select identity to edit
}
+ {selected.length === 1 && <> + + } { selected.length !== 0 && <> + + } + {selectedIdentities.length !== 0 && <> + }
@@ -618,6 +773,16 @@ const App = () => { {image === 0 &&
Select image to view
} {image !== 0 && } + {guess !== undefined && guess.identity &&
+ +
} diff --git a/server/db/photos.js b/server/db/photos.js index 564758a..c11e390 100755 --- a/server/db/photos.js +++ b/server/db/photos.js @@ -119,6 +119,7 @@ function init() { key: 'id', } }, + facesCount: Sequelize.INTEGER, descriptors: Sequelize.BLOB /* average of all faces mapped to this */ }, { timestamps: false diff --git a/server/routes/identities.js b/server/routes/identities.js index 297076a..e34cc68 100755 --- a/server/routes/identities.js +++ b/server/routes/identities.js @@ -31,7 +31,7 @@ const upsertIdentity = async(id, { firstName, lastName, middleName, - id + identityId: id }; if (id === -1 || !id) { @@ -41,7 +41,7 @@ const upsertIdentity = async(id, { 'VALUES(:displayName,:firstName,:lastName,:middleName)', { replacements: identity }); - identity.id = lastId; + identity.identityId = lastId; } else { await photoDB.sequelize.query( `UPDATE identities ` + @@ -50,7 +50,7 @@ const upsertIdentity = async(id, { 'firstName=:firstName, ' + 'lastName=:lastName, ' + 'middleName=:middleName ' + - 'WHERE id=:id', { + 'WHERE id=:identityId', { replacements: identity }); } @@ -74,7 +74,7 @@ const populateRelatedFaces = async (identity, count) => { "FROM faces " + "WHERE identityId=:identityId " + limit, { - replacements: { identityId: identity.id }, + replacements: { identityId: identity.identityId }, type: photoDB.Sequelize.QueryTypes.SELECT, raw: true }); @@ -92,13 +92,14 @@ const populateRelatedFaces = async (identity, count) => { } identity.relatedFaces.forEach(face => { - face.identityId = identity.id; + face.identityId = identity.identityId; face.distance = face.faceConfidence; face.descriptors = []; delete face.faceConfidence; }); } +/* Create new identity */ router.post('/', async (req, res) => { console.log(`POST ${req.url}`) if (!req.user.maintainer) { @@ -110,10 +111,12 @@ router.post('/', async (req, res) => { if (!identity) { return; } - populateRelatedFaces(identity, 1); + await updateIdentityFaces(identity); + await populateRelatedFaces(identity, 1); return res.status(200).send(identity); }); +/* Update identity */ router.put('/:id', async (req, res) => { console.log(`PUT ${req.url}`) if (!req.user.maintainer) { @@ -130,7 +133,8 @@ router.put('/:id', async (req, res) => { if (!identity) { return; } - populateRelatedFaces(identity); + await updateIdentityFaces(identity); + await populateRelatedFaces(identity); return res.status(200).send(identity); }); @@ -156,7 +160,7 @@ router.delete('/:id', async (req, res) => { await photoDB.sequelize.query( 'DELETE FROM identities ' + - 'WHERE identityId=:id', { + 'WHERE id=:id', { replacements: { id } }); @@ -164,11 +168,6 @@ router.delete('/:id', async (req, res) => { }); -const addFaceToIdentityDescriptors = (identity, face) => { -}; - -const removeFaceToIdentityDescriptors = (identity, face) => { -}; const writeIdentityDescriptors = async (identity) => { await photoDB.sequelize.query( @@ -180,6 +179,86 @@ const writeIdentityDescriptors = async (identity) => { ); }; + +/* Given a faceId, find the closest defined identity and return + * it as a guess -- does not modify the DB */ +router.get("/faces/guess/:faceId", async (req, res) => { + const faceId = parseInt(req.params.faceId); + if (faceId != req.params.faceId) { + return res.status(400).send({ message: "Invalid identity id." }); + } + + try { + const faces = await photoDB.sequelize.query( + "SELECT faces.*,faceDescriptors.* " + + "FROM faces,faceDescriptors " + + "WHERE faces.id=:faceId " + + "AND faceDescriptors.id=faces.descriptorId", { + replacements: { faceId }, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }); + + const identities = await photoDB.sequelize.query( + "SELECT * FROM identities", { + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }); + identities.forEach(x => { x.identityId = x.id; delete x.id;}); + + faces.forEach((face) => { + face.faceId = face.id; + delete face.id; + face.identityId = -1; + face.distance = -1; + face.identity = null; + + identities.forEach((identity) => { + if (!identity.descriptors) { + return; + } + + const distance = euclideanDistance( + face.descriptors, + identity.descriptors + ); + + if (face.identityId === -1) { + face.identityId = identity.identityId; + face.identity = identity; + face.distance = distance; + return; + } + + if (distance < face.distance) { + face.identityId = identity.identityId; + face.identity = identity; + face.distance = distance; + } + }); + }); + + /* Delete the VGG-Face descriptors and add relatedFaces[0] */ + const results = []; + await Promise.map(faces, async (face) => { + delete face.descriptors; + if (face.identity.descriptors) { + delete face.identity.descriptors; + } + if (!face.identity.relatedFaces) { + await populateRelatedFaces(face.identity, 1); + } + }, { + concurrency: 1 + }); + + return res.status(200).json(faces); + } catch (error) { + console.error(error); + return res.status(500).send("Error processing request."); + }; +}); + router.put("/faces/remove/:id", async (req, res) => { console.log(`PUT ${req.url}`) if (!req.user.maintainer) { @@ -206,13 +285,12 @@ router.put("/faces/remove/:id", async (req, res) => { } }); const identity = { - id: id, + identityId: id, faces: req.body.faces }; identity.faces = identity.faces.map(id => +id); - updateIdentityDescriptors(identity); - + await updateIdentityFaces(identity); return res.status(200).json(identity); } catch (error) { console.error(error); @@ -245,13 +323,12 @@ router.put("/faces/add/:id", async (req, res) => { } }); const identity = { - id: id, + identityId: id, faces: req.body.faces }; identity.faces = identity.faces.map(id => +id); - updateIdentityDescriptors(identity); - + await updateIdentityFaces(identity); return res.status(200).json(identity); } catch (error) { console.error(error); @@ -259,49 +336,6 @@ router.put("/faces/add/:id", async (req, res) => { }; }); -router.post("/", (req, res) => { - if (!req.user.maintainer) { - console.warn(`${req.user.name} attempted to modify photos.`); - return res.status(401).send("Unauthorized to modify photos."); - } - const identity = { - lastName: req.body.lastName || "", - firstName: req.body.firstName || "", - middleName: req.body.middleName || "" - }; - identity.name = req.body.name || (identity.firstName + " " + identity.lastName); - - let fields = []; - for (let key in identity) { - fields.push(key); - } - - if (!Array.isArray(req.body.faces) || req.body.faces.length == 0) { - return res.status(400).send("No faces supplied."); - } - - return photoDB.sequelize.query("INSERT INTO identities " + - "(" + fields.join(",") + ") " + - "VALUES(:" + fields.join(",:") + ")", { - replacements: identity, - }).then(([ results, metadata ]) => { - identity.id = metadata.lastID; - return photoDB.sequelize.query( - "UPDATE faces SET identityId=:identityId " + - "WHERE id IN (:faceIds)", { - replacements: { - identityId: identity.id, - faceIds: req.body.faces - } - }).then(() => { - identity.faces = req.body.faces; - return res.status(200).json([identity]); - }); - }).catch((error) => { - console.error(error); - return res.status(500).send("Error processing request."); - }); -}); function bufferToFloat32Array(buffer) { return new Float64Array(buffer.buffer, @@ -309,19 +343,24 @@ function bufferToFloat32Array(buffer) { buffer.byteLength / Float64Array.BYTES_PER_ELEMENT); } +function euclideanDistanceArray(a, b) { + let sum = 0; + for (let i = 0; i < a.length; i++) { + let delta = a[i] - b[i]; + sum += delta * delta; + } + return Math.sqrt(sum); +} + function euclideanDistance(a, b) { if (!a.buffer || !b.buffer) { return -1; } - let A = bufferToFloat32Array(a); - let B = bufferToFloat32Array(b); - let sum = 0; - for (let i = 0; i < A.length; i++) { - let delta = A[i] - B[i]; - sum += delta * delta; - } - return Math.sqrt(sum); + return euclideanDistanceArray( + bufferToFloat32Array(a), + bufferToFloat32Array(b) + ); } const getUnknownIdentity = async (faceCount) => { @@ -332,10 +371,11 @@ const getUnknownIdentity = async (faceCount) => { middleName: '', displayName: 'Unknown', descriptors: new Float32Array(0), - relatedFaces: [] + relatedFaces: [], + facesCount: 0 }; const limit = faceCount - ? ` LIMIT ${faceCount} ` + ? ` ORDER BY RANDOM() LIMIT ${faceCount} ` : ' ORDER BY faceConfidence DESC '; unknownIdentity.relatedFaces = await photoDB.sequelize.query( "SELECT id AS faceId,photoId,faceConfidence " + @@ -360,6 +400,13 @@ const getUnknownIdentity = async (faceCount) => { return unknownIdentity; } +const round = (x, precision) => { + if (precision === undefined) { + precision = 2; + } + return Number.parseFloat(x).toFixed(precision); +}; + /* Compute the identity's centroid descriptor from all faces * and determine closest face to that centroid. If either of * those values have changed, update the identity. @@ -369,8 +416,27 @@ const getUnknownIdentity = async (faceCount) => { */ const updateIdentityFaces = async (identity) => { if (!identity.identityId) { - identity.identityId = identity.id; + throw Error(`identityId is not set.`); } + + if (!identity.descriptors) { + const results = await photoDB.sequelize.query( + "SELECT " + + "descriptors,facesCount,faceId " + + "FROM identities " + + "WHERE id=:identityId", { + replacements: identity, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }); + Object.assign(identity, results[0]); + /* New identities do not have descriptors set */ + } + + if (identity.descriptors) { + identity.descriptors = bufferToFloat32Array(identity.descriptors); + } + const faces = await photoDB.sequelize.query( "SELECT " + "faces.*,faceDescriptors.* " + @@ -383,73 +449,132 @@ const updateIdentityFaces = async (identity) => { raw: true }); + if (faces.length === 0) { + return; + } + let average = undefined, closestId = -1, - closestDistance = -1, - count = 0; - + closestDistance = -1; + + /* First find the average centroid of all faces */ faces.forEach((face) => { - if (!identity.descriptors) { + /* Convert the descriptors from buffer to array so they can be + * modified */ + face.descriptors = bufferToFloat32Array(face.descriptors); + + /* First face starts the average sum */ + if (average === undefined) { + average = face.descriptors.slice(); return; } - if (!face.descriptors) { - return; - } - face.distance = euclideanDistance( + + /* Add this face descriptor into the average descriptor */ + for (let i = 0; i < face.descriptors.length; i++) { + average[i] = average[i] + face.descriptors[i]; + }; + }); + + /* Divide sum of descriptors to create average centroid */ + for (let i = 0; i < average.length; i++) { + average[i] = average[i] / faces.length; + } + + /* Now compute the distance from each face to the new centroid */ + faces.forEach((face) => { + let distance; + + distance = euclideanDistanceArray( face.descriptors, - identity.descriptors + average ); - face.descriptors = bufferToFloat32Array(face.descriptors).map(x => x * x); - if (closestId === -1) { - closestId = face.id; - closestDistance = face.distance; - average = descriptors; - count = 1; - return; - } - - descriptors.forEach((x, i) => { - average[i] += x; - }); - count++; - - if (face.distance < closestDistance) { - closestDistance = face.distance; + if (closestId === -1 || face.distance < closestDistance) { + closestDistance = distance; closestId = face.id; } }); - let same = true; - if (average) { - average = average.map(x => x / count); - same = bufferToFloat32Array(identity.descriptors) - .find((x, i) => average[i] === x) === undefined; - await Promise(faces, async (face) => { - const distance = euclideanDistanceArray(face.descriptors, average); - if (distance !== face.distance) { - await photoDB.sequelize.query( - 'UPDATE faces SET distance=:distance WHERE id=:faceId', { - replacements: face - } - ); - } - }); + /* Determine if the centroid for this identity has moved + * and for each relatedFace, update its distance to the centroid */ + if (!identity.descriptors) { + console.log(`Identity ${identity.identityId} has no descriptors`); } + let moved = (identity.descriptors === null ? 1 : 0) + || Number + .parseFloat(euclideanDistanceArray(identity.descriptors, average)) + .toFixed(4); + + /* If the average position has not changed, then face distances should + * not change either! */ + await Promise.map(faces, async (face) => { + /* All the buffer are already arrays, so use the short-cut version */ + const distance = Number + .parseFloat(euclideanDistanceArray(face.descriptors, average)) + .toFixed(4); + + if (Math.abs(distance - face.distance) > 0.0001) { + console.log( + `Updating face ${face.id} to ${round(distance, 2)} ` + + `(${distance - face.distance}) ` + + `from identity ${identity.identityId} (${identity.displayName})`); + face.distance = distance; + await photoDB.sequelize.query( + 'UPDATE faces SET distance=:distance WHERE id=:id', { + replacements: face + } + ); + } + }, { + concurrency: 1 + }); let sql = ''; + /* If there is a new closestId, then set the faceId field */ if (closestId !== -1 && closestId !== identity.faceId) { + console.log( + `Updating identity ${identity.identityId} closest face to ${closestId}`); sql = `${sql} faceId=:faceId`; + identity.faceId = closestId; } - if (!same) { + + /* If the centroid changed, update the identity descriptors to + * the new average */ + if (Math.abs(moved) > 0.0001) { + console.log( + `Updating identity ${identity.identityId} centroid ` + + `(moved ${Number.parseFloat(moved).toFixed(4)}).`); if (sql !== '') { sql = `${sql}, `; } sql = `${sql} descriptors=:descriptors`; - identity.descriptors = average; + // this: identity.descriptors = average; + // gives: Invalid value Float64Array(2622) + // + // this: identity.descriptors = new Blob(average); + // gives: Invalid value Blob { size: 54008, type: '' } + // + // this: identity.descriptors = Buffer.from(average); + // gives: all zeroes + // + // this: identity.descriptors = Buffer.from(average.buffer); + // gives: IT WORKS!!! + identity.descriptors = Buffer.from(average.buffer); } + + /* If the number of faces changed, update the facesCount */ + if (identity.facesCount !== faces.length) { + if (sql !== '') { + sql = `${sql}, `; + } + console.log( + `Updating identity ${identity.identityId} face count to ${faces.length}`); + identity.facesCount = faces.length; + sql = `${sql} facesCount=${faces.length}`; + } + + /* If any of the above required changes, actually commit to the DB */ if (sql !== '') { - identity.faceId = closestId; await photoDB.sequelize.query( `UPDATE identities SET ${sql} ` + `WHERE id=:identityId`, { @@ -487,7 +612,8 @@ router.get("/:id?", async (req, res) => { "identities.lastName," + "identities.middleName," + "identities.displayName," + - "identities.faceId " + + "identities.faceId," + + "identities.facesCount " + "FROM identities " + filter, { replacements: { id }, @@ -496,6 +622,9 @@ router.get("/:id?", async (req, res) => { }); await Promise.map(identities, async (identity) => { + console.log(`Updating ${identity.identityId}`); + await updateIdentityFaces(identity); + for (let field in identity) { if (field.match(/.*Name/) && identity[field] === null) { identity[field] = ''; @@ -526,6 +655,10 @@ router.get("/:id?", async (req, res) => { } ); + /* If this identity has at least one face associated with it, + * and it does not yet have the 'closest' face assigned, update + * the identity statistics. + */ if (identity.relatedFaces.length !== 0 && (!identity.faceId || identity.faceId === -1)) { await updateIdentityFaces(identity); @@ -541,6 +674,8 @@ router.get("/:id?", async (req, res) => { faceConfidence: 0 }); } + }, { + concurrency: 1 }); /* If no ID was provided then this call is returning