Added a few DB sanity tests for BLOB values between JS and Python

Signed-off-by: James Ketrenos <james_git@ketrenos.com>

client/src/App.css

@@ -21,12 +21,60 @@ div {
 
 .Identities {
   display: flex;
-  flex-grow: 1;
+  overflow-y: scroll;
+  flex-direction: column;
   border: 1px solid green;
 }
 
+.Identity {
+  display: flex;
+  flex-direction: column;
+  position: relative;
+  margin: 0.125rem;
+  border: 1px solid transparent;
+}
+
+.Identity:hover {
+  border: 1px solid yellow;
+  cursor: pointer;
+}
+
+.Identity .Title {
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  background-color: rgba(0, 0, 0, 0.5);
+  padding: 0.125rem;
+  font-size: 0.6rem;
+  color: white;
+}
+
+.Identity .Face {
+  width: 8rem;
+  height: 8rem;
+  background-size: contain !important;
+  background-repeat: no-repeat no-repeat !important;;
+  background-position: 50% 50% !important;;
+}
+
+
 .Cluster {
   display: flex;
+  flex-direction: column;
+  overflow-y: scroll;
   flex-grow: 1;
   border: 1px solid red;
+  padding: 0.5rem;
 }
+
+.Cluster .Info {
+  display: flex;
+  flex-direction: column;
+  align-items: flex-start;
+}
+
+.Cluster .Faces {
+  display: grid;
+  grid-template-columns: repeat(auto-fill, minmax(8rem, 1fr));
+}

client/src/App.tsx

@@ -3,10 +3,68 @@ import React, { useState, useMemo, useEffect } from 'react';
 import { useApi } from './useApi';
 import './App.css';
 
-const Cluster = () => {
+type ClusterProps = {
-  return (
+  id: number
-    <div className='Cluster'>cluster</div>
+};
+
+const Cluster = ({ id }: ClusterProps) => {
+  const [identity, setIdentity] = useState<Identity | undefined>(undefined);
+  const { loading, data } = useApi(
+    `../api/v1/identities/${id}`
   );
+
+  useEffect(() => {
+    if (data) {
+      if (Array.isArray(data) && data.length > 0) {
+        setIdentity(data[0] as Identity);
+      } else {
+        setIdentity(data as Identity);
+      }
+    }
+  }, [data]);
+
+  const relatedFacesJSX = useMemo(() => {
+    if (identity === undefined) {
+      return <></>;
+    }
+    return identity.relatedFaces.map((face) => {
+      const idPath = String(face.faceId % 100).padStart(2, '0');
+      return (
+        <div key={face.faceId}
+          className='Identity'>
+          <div className='Title'>
+            {face.distance}
+          </div>
+          <div className='Face'
+            style={{
+              background: `url("/faces/${idPath}/${face.faceId}.jpg")`,
+            }} />
+        </div>
+      );
+    });
+  }, [identity]);
+
+  return (
+    <div className='Cluster'>
+      { loading && `Loading ${id}`}
+      { identity !== undefined && <div className="Info">
+        <div>{identity.lastName}</div>
+        <div>{identity.firstName}</div>
+        <div>{identity.middleName}</div>
+        <div>{identity.displayName}</div>
+        <div>Faces: {identity.relatedFaces.length}</div>
+      </div> }
+      { identity !== undefined && <div className="Faces">
+        { relatedFacesJSX }
+      </div> }
+    </div>
+  );
+};
+
+type Face = {
+  distance: number,
+  faceId: number,
+  photoId: number
 };
 
 type Identity = {
@@ -16,22 +74,41 @@ type Identity = {
   descriptors: number[],
   id: number
   displayName: string,
+  relatedFaces: Face[]
 };
 
 interface IdentitiesProps {
+  setIdentity?(id: number): void,
   identities: Identity[]
 };
 
-const Identities = ({ identities } : IdentitiesProps) => {
+const Identities = ({ identities, setIdentity } : IdentitiesProps) => {
-  const identitiesJSX = useMemo(() => 
+
-    identities.map((identity) => {
+
-      const idPath = String(identity.id % 100).padStart(2, '0');
+  const identitiesJSX = useMemo(() => {
-      return (<img 
+    const loadIdentity = (id: number): void => {
-        key={identity.id} 
+      if (setIdentity) {
-        alt={identity.id.toString()} 
+        setIdentity(id)
-        src={`/faces/${idPath}/${identity.id}.jpg`}/>);
+      }
-    }
+    };
-  ), [ identities ]);
+    return  identities.map((identity) => {
+      const face = identity.relatedFaces[0];
+      const idPath = String(face.faceId % 100).padStart(2, '0');
+      return (
+        <div key={face.faceId} 
+          onClick={() => loadIdentity(identity.id)}
+          className='Identity'>
+          <div className='Title'>
+            {identity.displayName}
+          </div>
+          <div className='Face'
+            style={{
+              background: `url("/faces/${idPath}/${face.faceId}.jpg")`,
+            }}/>
+        </div>
+      );
+    });
+  }, [ setIdentity, identities ]);
 
   return (
     <div className='Identities'>
@@ -42,8 +119,9 @@ const Identities = ({ identities } : IdentitiesProps) => {
 
 const App = () => {
   const [identities, setIdentities] = useState<Identity[]>([]);
+  const [identity, setIdentity] = useState<number>(0);
   const { loading, data } = useApi(
-    '../api/v1/faces'
+    '../api/v1/identities'
   );
 
   useEffect(() => {
@@ -56,9 +134,12 @@ const App = () => {
     <div className="App">
       <div className="Worksheet">
         { loading && <div>Loading...</div> }
+        { !loading && identity !== 0 && <Cluster id={identity} />}
+        { !loading && identity === 0 && <div className="Cluster">
+          Select identity to edit
+        </div> }
         { !loading && <>
-          <Cluster/>
+          <Identities {... {identities, setIdentity }}/>
-          <Identities identities={identities}/>
         </> }
       </div>
     </div>

client/src/useApi.tsx

@@ -8,12 +8,17 @@ type UseApi = {
 };
 
 const useApi = (_url: string, _options?: {}) : UseApi => {
-  const [loading, setLoading] = useState(true);
+  const [loading, setLoading] = useState(false);
   const [data, setData] = useState(undefined);
   const [error, setError] = useState<any>(undefined);
 
   useEffect(() => {
+    if (_url === '' || loading) {
+      return;
+    }
     const fetchApi = async () => {
+      console.log(`Fetching ${_url}...`);
+      setLoading(true);
       try {
         const res = await window.fetch(_url, _options);
         const data = await res.json();
@@ -27,7 +32,7 @@ const useApi = (_url: string, _options?: {}) : UseApi => {
     };
 
     fetchApi();
-  }, [_url, _options]);
+  }, [_url, _options, loading]);
 
   return { loading, data, error };
 };

ketrface/db-test.py

@@ -0,0 +1,55 @@
+
+import functools
+
+from ketrface.util import *
+from ketrface.dbscan import *
+from ketrface.db import *
+from ketrface.config import *
+
+config = read_config()
+
+html_path = merge_config_path(config['path'], 'frontend')
+pictures_path = merge_config_path(config['path'], config['picturesPath'])
+faces_path = merge_config_path(config['path'], config['facesPath'])
+db_path = merge_config_path(config['path'], config["db"]["photos"]["host"])
+html_base = config['basePath']
+if html_base == "/":
+  html_base = "."
+
+print(f'Connecting to database: {db_path}')
+conn = create_connection(db_path)
+with conn:
+  cur = conn.cursor()
+  res = cur.execute('''
+    SELECT identities.descriptors,
+    GROUP_CONCAT(faces.id) AS relatedFaceIds,
+    GROUP_CONCAT(faces.descriptorId) AS relatedFaceDescriptorIds,
+    GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds 
+    FROM identities 
+    INNER JOIN faces ON identities.id=faces.identityId
+    WHERE identities.id=7
+    GROUP BY identities.id
+    ''')
+  for identity in res.fetchall():
+    relatedFaceDescriptorIds = identity[2].split(',')
+  
+    res2 = cur.execute(
+      'SELECT descriptors FROM facedescriptors WHERE id IN (%s)' %
+      ','.join('?'*len(relatedFaceDescriptorIds)), relatedFaceDescriptorIds)
+
+    descriptors = []
+    for row2 in res2.fetchall():
+      descriptors.append(np.frombuffer(row2[0]))
+
+    distances = []
+
+    relatedFaceIds = identity[2].split(',')
+    for i, face in enumerate(relatedFaceIds):
+      distance = findEuclideanDistance(
+        descriptors[i],
+        np.frombuffer(identity[0])
+      )
+      distances.append(distance)
+
+    distances.sort()
+    print(distances)

scripts/kill-server.sh

@@ -1,10 +1,13 @@
-
 #!/bin/bash
 pid=$(ps aux | 
-  grep '[0-9] node app.js' |
+  grep -E '[0-9] (/usr/bin/)?node .*server/app.js' |
   while read user pid rest; do 
     echo $pid; 
   done)
 if [[ "$pid" != "" ]]; then
+  echo "Killing ${pid}"
   kill $pid
+else
+  echo "No node server found"
 fi
+

server/db-test.js

@@ -0,0 +1,98 @@
+"use strict";
+
+const Promise = require("bluebird");
+
+let photoDB;
+
+function bufferToFloat32Array(buffer) {
+  return new Float64Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / Float64Array.BYTES_PER_ELEMENT);
+}
+
+function euclideanDistance(a, b) {
+  let A = bufferToFloat32Array(a);
+  let B = bufferToFloat32Array(b);
+  console.log(A.length, B.length);
+  let sum = 0;
+  for (let i = 0; i < A.length; i++) {
+    let delta = A[i] - B[i];
+    sum += delta * delta;
+  }
+  return Math.sqrt(sum);
+}
+
+require("./db/photos").then(function(db) {
+  photoDB = db;
+})
+.then(async () => {
+  const id = 7;
+  const filter = ` WHERE identities.id=:id `;
+  const identities = await photoDB.sequelize.query("SELECT " +
+    "identities.*," +
+    "GROUP_CONCAT(faces.id) AS relatedFaceIds," +
+    "GROUP_CONCAT(faces.descriptorId) AS relatedFaceDescriptorIds," +
+    "GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds " +
+    "FROM identities " +
+    "INNER JOIN faces ON identities.id=faces.identityId " +
+    filter +
+    "GROUP BY identities.id", {
+    replacements: { id },
+    type: photoDB.Sequelize.QueryTypes.SELECT,
+    raw: true
+  });
+
+  await Promise.map(identities, async (identity) => {
+    [ 'firstName', 'middleName', 'lastName' ].forEach(key => {
+      if (!identity[key]) {
+        identity[key] = '';
+      }
+    });
+
+    const relatedFaces = identity.relatedFaceIds.split(","),
+      relatedFacePhotos = identity.relatedFacePhotoIds.split(",");
+
+    let descriptors = await photoDB.sequelize.query(
+      `SELECT descriptors FROM facedescriptors WHERE id in (:ids)`, {
+        replacements: { 
+          ids: identity.relatedFaceDescriptorIds.split(',') 
+        },
+        type: photoDB.Sequelize.QueryTypes.SELECT,
+        raw: true
+      }
+    );
+    
+    descriptors = descriptors.map(entry => entry.descriptors);
+ 
+    identity.relatedFaces = relatedFaces.map((faceId, index) => {
+      const distance = euclideanDistance(
+        descriptors[index],
+        identity.descriptors
+      );
+
+      console.log(index, distance);
+      return {
+        faceId,
+        photoId: relatedFacePhotos[index],
+        distance
+      };
+    });
+
+    identity
+      .relatedFaces
+      .sort((A, B) => {
+        return A.distance - B.distance;
+      });
+
+    /* If no filter was specified, only return the best face for
+     * the identity */
+    if (!filter) {
+      identity.relatedFaces = [ identity.relatedFaces[0] ];
+    }
+
+    delete identity.descriptors;
+    delete identity.relatedFaceIds;
+    delete identity.relatedFacePhotoIds;
+    delete identity.relatedIdentityDescriptors;
+  }, {
+    concurrency: 10
+  });
+});

server/routes/identities.js

@@ -90,7 +90,7 @@ router.post("/", (req, res) => {
 });
 
 function bufferToFloat32Array(buffer) {
-  return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / Float32Array.BYTES_PER_ELEMENT);
+  return new Float64Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / Float64Array.BYTES_PER_ELEMENT);
 }
 
 function euclideanDistance(a, b) {
@@ -105,6 +105,8 @@ function euclideanDistance(a, b) {
 }
 
 router.get("/:id?", async (req, res) => {
+  console.log(`GET ${req.url}`);
+
   let id;
 
   if (req.params.id) {
@@ -119,10 +121,9 @@ router.get("/:id?", async (req, res) => {
   const identities = await photoDB.sequelize.query("SELECT " +
     "identities.*," +
     "GROUP_CONCAT(faces.id) AS relatedFaceIds," +
-    "GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds," +
+    "GROUP_CONCAT(faces.descriptorId) AS relatedFaceDescriptorIds," +
-    "GROUP_CONCAT(facedescriptors.descriptors) AS relatedIdentityDescriptors " +
+    "GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds " +
-    "FROM identities " + 
+    "FROM identities " +
-    "INNER JOIN facedescriptors ON facedescriptors.id=faces.descriptorId " +
     "INNER JOIN faces ON identities.id=faces.identityId " +
     filter +
     "GROUP BY identities.id", {
@@ -131,7 +132,7 @@ router.get("/:id?", async (req, res) => {
     raw: true
   });
   
-  identities.forEach((identity) => {
+  await Promise.map(identities, async (identity) => {
     [ 'firstName', 'middleName', 'lastName' ].forEach(key => {
       if (!identity[key]) {
         identity[key] = '';
@@ -139,15 +140,26 @@ router.get("/:id?", async (req, res) => {
     });
 
     const relatedFaces = identity.relatedFaceIds.split(","),
-      relatedFacePhotos = identity.relatedFacePhotoIds.split(","),
+      relatedFacePhotos = identity.relatedFacePhotoIds.split(",");
-      relatedIdentityDescriptors = 
+
-        identity.relatedIdentityDescriptors.split(",");
+    let descriptors = await photoDB.sequelize.query(
+      `SELECT descriptors FROM facedescriptors WHERE id in (:ids)`, {
+        replacements: {
+          ids: identity.relatedFaceDescriptorIds.split(',')
+        },
+        type: photoDB.Sequelize.QueryTypes.SELECT,
+        raw: true
+      }
+    );
+     
+    descriptors = descriptors.map(entry => entry.descriptors);
 
     identity.relatedFaces = relatedFaces.map((faceId, index) => {
       const distance = euclideanDistance(
-        relatedIdentityDescriptors[index],
+        descriptors[index],
         identity.descriptors
       );
+
       return {
         faceId,
         photoId: relatedFacePhotos[index],
@@ -155,61 +167,24 @@ router.get("/:id?", async (req, res) => {
       };
     });
 
+    identity
+      .relatedFaces
+      .sort((A, B) => {
+        return A.distance - B.distance;
+      });
+
+    /* If no filter was specified, only return the best face for
+     * the identity */
+    if (!filter) {
+      identity.relatedFaces = [ identity.relatedFaces[0] ];
+    }
+
+    delete identity.descriptors;
     delete identity.relatedFaceIds;
     delete identity.relatedFacePhotoIds;
     delete identity.relatedIdentityDescriptors;
   });
 
-    //if (!req.query.withScore) {
-  console.log("No score request.");
-  return res.status(200).json(identities);
-    //}
-
-  // THe rest of this routine needs to be reworked -- I don't
-  // recall what it was doing; maybe getting a list of all identities
-  // sorted with distance to this faceId?
-  console.log("Looking up score against: " + req.query.withScore);
-
-  await Promise.map(identities, async (identity) => {
-    const descriptors = photoDB.sequelize.query(
-      "SELECT id FROM facedescriptors " +
-      "WHERE descriptorId " +
-      "IN (:id,:descriptorIds)", {
-      replacements: {
-        id: parseInt(req.query.withScore),
-        descriptorIds: identity.relatedFaces.map(
-          face => parseInt(face.faceId))
-      },
-      type: photoDB.Sequelize.QueryTypes.SELECT,
-      raw: true
-    });
-    let target;
-    for (let i = 0; i < descriptors.length; i++) {
-      if (descriptors[i].descriptorId == req.query.withScore) {
-        target = descriptors[i].descriptors;
-        break;
-      }
-    }
-    if (!target) {
-      console.warn("Could not find descriptor for requested face: " + req.query.withScore);
-      return;
-    }
-
-    /* For each face's descriptor returned for this identity, compute the distance between the
-      * requested photo and that face descriptor */
-    descriptors.forEach((descriptor) => {
-      for (let i = 0; i < identity.relatedFaces.length; i++) {
-        if (identity.relatedFaces[i].faceId == descriptor.faceId) {
-          identity.relatedFaces[i].distance = euclideanDistance(target, descriptor.descriptors);
-          identity.relatedFaces[i].descriptors = descriptor.descriptors;
-          return;
-        }
-      }
-    });
-  }, {
-    concurrency: 5
-  });
-
   return res.status(200).json(identities);
 });