diff --git a/client/src/App.css b/client/src/App.css index e129486..09fc090 100644 --- a/client/src/App.css +++ b/client/src/App.css @@ -21,12 +21,60 @@ div { .Identities { display: flex; - flex-grow: 1; + overflow-y: scroll; + flex-direction: column; border: 1px solid green; } +.Identity { + display: flex; + flex-direction: column; + position: relative; + margin: 0.125rem; + border: 1px solid transparent; +} + +.Identity:hover { + border: 1px solid yellow; + cursor: pointer; +} + +.Identity .Title { + position: absolute; + top: 0; + left: 0; + right: 0; + background-color: rgba(0, 0, 0, 0.5); + padding: 0.125rem; + font-size: 0.6rem; + color: white; +} + +.Identity .Face { + width: 8rem; + height: 8rem; + background-size: contain !important; + background-repeat: no-repeat no-repeat !important;; + background-position: 50% 50% !important;; +} + + .Cluster { display: flex; + flex-direction: column; + overflow-y: scroll; flex-grow: 1; border: 1px solid red; + padding: 0.5rem; } + +.Cluster .Info { + display: flex; + flex-direction: column; + align-items: flex-start; +} + +.Cluster .Faces { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(8rem, 1fr)); +} \ No newline at end of file diff --git a/client/src/App.tsx b/client/src/App.tsx index 3a17c77..769f041 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -3,10 +3,68 @@ import React, { useState, useMemo, useEffect } from 'react'; import { useApi } from './useApi'; import './App.css'; -const Cluster = () => { - return ( -
cluster
+type ClusterProps = { + id: number +}; + +const Cluster = ({ id }: ClusterProps) => { + const [identity, setIdentity] = useState(undefined); + const { loading, data } = useApi( + `../api/v1/identities/${id}` ); + + useEffect(() => { + if (data) { + if (Array.isArray(data) && data.length > 0) { + setIdentity(data[0] as Identity); + } else { + setIdentity(data as Identity); + } + } + }, [data]); + + const relatedFacesJSX = useMemo(() => { + if (identity === undefined) { + return <>; + } + return identity.relatedFaces.map((face) => { + const idPath = String(face.faceId % 100).padStart(2, '0'); + return ( +
+
+ {face.distance} +
+
+
+ ); + }); + }, [identity]); + + return ( +
+ { loading && `Loading ${id}`} + { identity !== undefined &&
+
{identity.lastName}
+
{identity.firstName}
+
{identity.middleName}
+
{identity.displayName}
+
Faces: {identity.relatedFaces.length}
+
} + { identity !== undefined &&
+ { relatedFacesJSX } +
} +
+ ); +}; + +type Face = { + distance: number, + faceId: number, + photoId: number }; type Identity = { @@ -16,22 +74,41 @@ type Identity = { descriptors: number[], id: number displayName: string, + relatedFaces: Face[] }; interface IdentitiesProps { + setIdentity?(id: number): void, identities: Identity[] }; -const Identities = ({ identities } : IdentitiesProps) => { - const identitiesJSX = useMemo(() => - identities.map((identity) => { - const idPath = String(identity.id % 100).padStart(2, '0'); - return ({identity.id.toString()}); - } - ), [ identities ]); +const Identities = ({ identities, setIdentity } : IdentitiesProps) => { + + + const identitiesJSX = useMemo(() => { + const loadIdentity = (id: number): void => { + if (setIdentity) { + setIdentity(id) + } + }; + return identities.map((identity) => { + const face = identity.relatedFaces[0]; + const idPath = String(face.faceId % 100).padStart(2, '0'); + return ( +
loadIdentity(identity.id)} + className='Identity'> +
+ {identity.displayName} +
+
+
+ ); + }); + }, [ setIdentity, identities ]); return (
@@ -42,8 +119,9 @@ const Identities = ({ identities } : IdentitiesProps) => { const App = () => { const [identities, setIdentities] = useState([]); + const [identity, setIdentity] = useState(0); const { loading, data } = useApi( - '../api/v1/faces' + '../api/v1/identities' ); useEffect(() => { @@ -56,9 +134,12 @@ const App = () => {
{ loading &&
Loading...
} + { !loading && identity !== 0 && } + { !loading && identity === 0 &&
+ Select identity to edit +
} { !loading && <> - - + }
diff --git a/client/src/useApi.tsx b/client/src/useApi.tsx index 377f9ca..ef6d04a 100644 --- a/client/src/useApi.tsx +++ b/client/src/useApi.tsx @@ -8,12 +8,17 @@ type UseApi = { }; const useApi = (_url: string, _options?: {}) : UseApi => { - const [loading, setLoading] = useState(true); + const [loading, setLoading] = useState(false); const [data, setData] = useState(undefined); const [error, setError] = useState(undefined); useEffect(() => { + if (_url === '' || loading) { + return; + } const fetchApi = async () => { + console.log(`Fetching ${_url}...`); + setLoading(true); try { const res = await window.fetch(_url, _options); const data = await res.json(); @@ -27,7 +32,7 @@ const useApi = (_url: string, _options?: {}) : UseApi => { }; fetchApi(); - }, [_url, _options]); + }, [_url, _options, loading]); return { loading, data, error }; }; diff --git a/ketrface/db-test.py b/ketrface/db-test.py new file mode 100644 index 0000000..1513529 --- /dev/null +++ b/ketrface/db-test.py @@ -0,0 +1,55 @@ + +import functools + +from ketrface.util import * +from ketrface.dbscan import * +from ketrface.db import * +from ketrface.config import * + +config = read_config() + +html_path = merge_config_path(config['path'], 'frontend') +pictures_path = merge_config_path(config['path'], config['picturesPath']) +faces_path = merge_config_path(config['path'], config['facesPath']) +db_path = merge_config_path(config['path'], config["db"]["photos"]["host"]) +html_base = config['basePath'] +if html_base == "/": + html_base = "." + +print(f'Connecting to database: {db_path}') +conn = create_connection(db_path) +with conn: + cur = conn.cursor() + res = cur.execute(''' + SELECT identities.descriptors, + GROUP_CONCAT(faces.id) AS relatedFaceIds, + GROUP_CONCAT(faces.descriptorId) AS relatedFaceDescriptorIds, + GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds + FROM identities + INNER JOIN faces ON identities.id=faces.identityId + WHERE identities.id=7 + GROUP BY identities.id + ''') + for identity in res.fetchall(): + relatedFaceDescriptorIds = identity[2].split(',') + + res2 = cur.execute( + 'SELECT descriptors FROM facedescriptors WHERE id IN (%s)' % + ','.join('?'*len(relatedFaceDescriptorIds)), relatedFaceDescriptorIds) + + descriptors = [] + for row2 in res2.fetchall(): + descriptors.append(np.frombuffer(row2[0])) + + distances = [] + + relatedFaceIds = identity[2].split(',') + for i, face in enumerate(relatedFaceIds): + distance = findEuclideanDistance( + descriptors[i], + np.frombuffer(identity[0]) + ) + distances.append(distance) + + distances.sort() + print(distances) \ No newline at end of file diff --git a/scripts/kill-server.sh b/scripts/kill-server.sh index 7e9a4d5..c4101fb 100755 --- a/scripts/kill-server.sh +++ b/scripts/kill-server.sh @@ -1,10 +1,13 @@ - #!/bin/bash pid=$(ps aux | - grep '[0-9] node app.js' | + grep -E '[0-9] (/usr/bin/)?node .*server/app.js' | while read user pid rest; do echo $pid; done) if [[ "$pid" != "" ]]; then + echo "Killing ${pid}" kill $pid +else + echo "No node server found" fi + diff --git a/server/db-test.js b/server/db-test.js new file mode 100644 index 0000000..cbeaee7 --- /dev/null +++ b/server/db-test.js @@ -0,0 +1,98 @@ +"use strict"; + +const Promise = require("bluebird"); + +let photoDB; + +function bufferToFloat32Array(buffer) { + return new Float64Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / Float64Array.BYTES_PER_ELEMENT); +} + +function euclideanDistance(a, b) { + let A = bufferToFloat32Array(a); + let B = bufferToFloat32Array(b); + console.log(A.length, B.length); + let sum = 0; + for (let i = 0; i < A.length; i++) { + let delta = A[i] - B[i]; + sum += delta * delta; + } + return Math.sqrt(sum); +} + +require("./db/photos").then(function(db) { + photoDB = db; +}) +.then(async () => { + const id = 7; + const filter = ` WHERE identities.id=:id `; + const identities = await photoDB.sequelize.query("SELECT " + + "identities.*," + + "GROUP_CONCAT(faces.id) AS relatedFaceIds," + + "GROUP_CONCAT(faces.descriptorId) AS relatedFaceDescriptorIds," + + "GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds " + + "FROM identities " + + "INNER JOIN faces ON identities.id=faces.identityId " + + filter + + "GROUP BY identities.id", { + replacements: { id }, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }); + + await Promise.map(identities, async (identity) => { + [ 'firstName', 'middleName', 'lastName' ].forEach(key => { + if (!identity[key]) { + identity[key] = ''; + } + }); + + const relatedFaces = identity.relatedFaceIds.split(","), + relatedFacePhotos = identity.relatedFacePhotoIds.split(","); + + let descriptors = await photoDB.sequelize.query( + `SELECT descriptors FROM facedescriptors WHERE id in (:ids)`, { + replacements: { + ids: identity.relatedFaceDescriptorIds.split(',') + }, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + } + ); + + descriptors = descriptors.map(entry => entry.descriptors); + + identity.relatedFaces = relatedFaces.map((faceId, index) => { + const distance = euclideanDistance( + descriptors[index], + identity.descriptors + ); + + console.log(index, distance); + return { + faceId, + photoId: relatedFacePhotos[index], + distance + }; + }); + + identity + .relatedFaces + .sort((A, B) => { + return A.distance - B.distance; + }); + + /* If no filter was specified, only return the best face for + * the identity */ + if (!filter) { + identity.relatedFaces = [ identity.relatedFaces[0] ]; + } + + delete identity.descriptors; + delete identity.relatedFaceIds; + delete identity.relatedFacePhotoIds; + delete identity.relatedIdentityDescriptors; + }, { + concurrency: 10 + }); +}); \ No newline at end of file diff --git a/server/routes/identities.js b/server/routes/identities.js index b264847..45a5ca4 100755 --- a/server/routes/identities.js +++ b/server/routes/identities.js @@ -90,7 +90,7 @@ router.post("/", (req, res) => { }); function bufferToFloat32Array(buffer) { - return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / Float32Array.BYTES_PER_ELEMENT); + return new Float64Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / Float64Array.BYTES_PER_ELEMENT); } function euclideanDistance(a, b) { @@ -105,6 +105,8 @@ function euclideanDistance(a, b) { } router.get("/:id?", async (req, res) => { + console.log(`GET ${req.url}`); + let id; if (req.params.id) { @@ -119,10 +121,9 @@ router.get("/:id?", async (req, res) => { const identities = await photoDB.sequelize.query("SELECT " + "identities.*," + "GROUP_CONCAT(faces.id) AS relatedFaceIds," + - "GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds," + - "GROUP_CONCAT(facedescriptors.descriptors) AS relatedIdentityDescriptors " + - "FROM identities " + - "INNER JOIN facedescriptors ON facedescriptors.id=faces.descriptorId " + + "GROUP_CONCAT(faces.descriptorId) AS relatedFaceDescriptorIds," + + "GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds " + + "FROM identities " + "INNER JOIN faces ON identities.id=faces.identityId " + filter + "GROUP BY identities.id", { @@ -131,7 +132,7 @@ router.get("/:id?", async (req, res) => { raw: true }); - identities.forEach((identity) => { + await Promise.map(identities, async (identity) => { [ 'firstName', 'middleName', 'lastName' ].forEach(key => { if (!identity[key]) { identity[key] = ''; @@ -139,15 +140,26 @@ router.get("/:id?", async (req, res) => { }); const relatedFaces = identity.relatedFaceIds.split(","), - relatedFacePhotos = identity.relatedFacePhotoIds.split(","), - relatedIdentityDescriptors = - identity.relatedIdentityDescriptors.split(","); + relatedFacePhotos = identity.relatedFacePhotoIds.split(","); + + let descriptors = await photoDB.sequelize.query( + `SELECT descriptors FROM facedescriptors WHERE id in (:ids)`, { + replacements: { + ids: identity.relatedFaceDescriptorIds.split(',') + }, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + } + ); + + descriptors = descriptors.map(entry => entry.descriptors); identity.relatedFaces = relatedFaces.map((faceId, index) => { const distance = euclideanDistance( - relatedIdentityDescriptors[index], + descriptors[index], identity.descriptors ); + return { faceId, photoId: relatedFacePhotos[index], @@ -155,61 +167,24 @@ router.get("/:id?", async (req, res) => { }; }); + identity + .relatedFaces + .sort((A, B) => { + return A.distance - B.distance; + }); + + /* If no filter was specified, only return the best face for + * the identity */ + if (!filter) { + identity.relatedFaces = [ identity.relatedFaces[0] ]; + } + + delete identity.descriptors; delete identity.relatedFaceIds; delete identity.relatedFacePhotoIds; delete identity.relatedIdentityDescriptors; }); - //if (!req.query.withScore) { - console.log("No score request."); - return res.status(200).json(identities); - //} - - // THe rest of this routine needs to be reworked -- I don't - // recall what it was doing; maybe getting a list of all identities - // sorted with distance to this faceId? - console.log("Looking up score against: " + req.query.withScore); - - await Promise.map(identities, async (identity) => { - const descriptors = photoDB.sequelize.query( - "SELECT id FROM facedescriptors " + - "WHERE descriptorId " + - "IN (:id,:descriptorIds)", { - replacements: { - id: parseInt(req.query.withScore), - descriptorIds: identity.relatedFaces.map( - face => parseInt(face.faceId)) - }, - type: photoDB.Sequelize.QueryTypes.SELECT, - raw: true - }); - let target; - for (let i = 0; i < descriptors.length; i++) { - if (descriptors[i].descriptorId == req.query.withScore) { - target = descriptors[i].descriptors; - break; - } - } - if (!target) { - console.warn("Could not find descriptor for requested face: " + req.query.withScore); - return; - } - - /* For each face's descriptor returned for this identity, compute the distance between the - * requested photo and that face descriptor */ - descriptors.forEach((descriptor) => { - for (let i = 0; i < identity.relatedFaces.length; i++) { - if (identity.relatedFaces[i].faceId == descriptor.faceId) { - identity.relatedFaces[i].distance = euclideanDistance(target, descriptor.descriptors); - identity.relatedFaces[i].descriptors = descriptor.descriptors; - return; - } - } - }); - }, { - concurrency: 5 - }); - return res.status(200).json(identities); });