Functional for editing clusters; need to add merging

Signed-off-by: James Ketrenos <james_git@ketrenos.com>

README.md

@@ -8,6 +8,26 @@ This photo manager performs the following:
 3. Schedule backend processing of all photos that have not been face 
    scanned with the latest FACE_SCANNER version
 
+## Populating...
+
+1. App builds photo DB. 
+    This occurs via 'server/app.js' (npm start)
+2. Faces are identified in all photos. 
+    This occurs in 'ketrface/detect.py' (python3 detect.py)
+  1. mtcnn finds faces (threshold > 0.95)
+  2. vgg-face is used to generate detectors
+3. Faces are clustered into groups. 
+    This occurs in 'ketrface/cluster.py' (python3 cluster.py)
+  1. DBSCAN generates first set of clusters
+  2. Centroid calculated for all clusters
+  3. Faces are pruned from clusters if they exceed a threshold
+  4. Clusters are merged based on centroid distances
+4. Clusters are examined to ensure only one face per image is assigned an identity.
+    This occurs in 'ketrface/split.py' (python3 cluster.py)
+  1. echo "select f1.id,f2.id,f1.photoId,f1.identityId from faces as f1 join faces as f2 on f2.identityId=f1.identityId and f1.photoId=f2.photoId and f1.id!=f2.id;"
+  2. For each face in the same photo, determine which face is closest to the cluster centroid. Move the other to cluster-1.{N} where N increases for each duplicate
+
+
 # To use the Docker
 
 Edit the environment file '.env' and set PICTURES to the correct

client/src/App.css

@@ -36,12 +36,14 @@ div {
 }
 
 .Identities {
-  display: flex;
+  display: grid;
-  padding: 0.25rem;
+  user-select: none;
-  gap: 0.25rem;
   overflow-y: scroll;
-  flex-direction: column;
+  overflow-x: clip;
-  border: 1px solid green;
+  height: 100%;
+  width: 100%;
+  gap: 0.25rem;
+  grid-template-columns: repeat(auto-fill, minmax(8.5rem, auto));
 }
 
 .Face {
@@ -49,11 +51,12 @@ div {
   box-sizing: border-box;
   flex-direction: column;
   position: relative;
-  border: 2px solid #444;
+  border: 0.25rem solid #444;
 }
 
 .Face:hover {
   cursor: pointer;
+  border-color: yellow;
 }
 
 .ClusterEditor {
@@ -82,17 +85,19 @@ div {
   background-position: 50% 50% !important;
 }
 
-.Active {
+.IdentityForm {
-  filter: brightness(1.25);
+  display: grid;
-  border-color: orange !important;
+  grid-template-columns: 1fr 1fr;
 }
 
-.Face:hover {
+.Face.Active,
-  border-color: yellow;
+.FaceBox.Active {
+  filter: brightness(1.25);
+  border-color: orange;
 }
 
 .Face.Selected {
-  border-color: blue;
+  border-color: blue !important;
 }
 
 .Face .Title {

client/src/App.tsx

@@ -1,5 +1,5 @@
 
-import React, { useState, useMemo, useEffect, useRef } from 'react';
+import React, { useState, useMemo, useEffect, useRef, useCallback } from 'react';
 import { useApi } from './useApi';
 import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels";
 import {
@@ -66,10 +66,11 @@ const Photo = ({ photoId, onFaceClick }: any) => {
     return makeFaceBoxes(image, dimensions, onFaceClick);
   }, [image, dimensions, onFaceClick]);
 
-  useEffect(() => {
+  const checkResize = useCallback(() => {
     if (!ref.current) {
       return;
     }
+
     const el: Element = ref.current as Element;
     if (dimensions.height !== el.clientHeight
       || dimensions.width !== el.clientWidth) {
@@ -78,7 +79,13 @@ const Photo = ({ photoId, onFaceClick }: any) => {
         width: el.clientWidth
       })
     }
-  }/*, [dimensions.height, dimensions.width]*/);
+  }, [setDimensions, dimensions]);
+
+ 
+  useEffect(() => {
+    let timer = setInterval(() => checkResize(), 250);
+    return () => { clearInterval(timer); }
+  }, [checkResize]);
 
   useEffect(() => {
     if (photoId === 0) {
@@ -98,11 +105,16 @@ const Photo = ({ photoId, onFaceClick }: any) => {
     return <></>
   }
 
-  return (<div className="Image"
+  return (<div className="Image" ref={ref}>
-    ref={ref}
+    <img 
-    style={{
+      src={`${base}/../${image.path}thumbs/scaled/${image.filename}`.replace(/ /g, '%20')}
-      background: `url("${base}/../${image.path}thumbs/scaled/${image.filename}")`.replace(/ /g, '%20')
+      style={{
-    }}>{ faces }</div>
+        objectFit: 'contain',
+        width: '100%',
+        height: '100%'
+      }} />
+      { faces }
+    </div>
   );
 };
 
@@ -113,7 +125,10 @@ const onFaceMouseEnter = (e: any, face: FaceData) => {
   if (face.identity) {
     const identityId = face.identity.identityId;
     els.splice(0, 0, 
-      ...document.querySelectorAll(`[data-identity-id="${identityId}"]`));
+      ...document.querySelectorAll(
+        `.Identities [data-identity-id="${identityId}"]`),
+      ...document.querySelectorAll(
+        `.Photo [data-identity-id="${identityId}"]`));
   }
 
   els.forEach(el => {
@@ -148,10 +163,13 @@ const Face = ({ face, onFaceClick, title, ...rest }: any) => {
       onMouseEnter={(e) => { onFaceMouseEnter(e, face) }}
       onMouseLeave={(e) => { onFaceMouseLeave(e, face) }}
       className='Face'>
-      <div className='Image'
+      <div className='Image'>
-        style={{
+        <img src={`${base}/../faces/${idPath}/${faceId}.jpg`}
-          background: `url("${base}/../faces/${idPath}/${faceId}.jpg")`,
+          style={{
-        }}>
+            objectFit: 'contain',
+            width: '100%',
+            height: '100%'
+          }}/>
         <div className='Title'>{title}</div>
       </div>
     </div>
@@ -166,6 +184,8 @@ type ClusterProps = {
 };
 
 const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps) => {
+  console.log(identity);
+
   const relatedFacesJSX = useMemo(() => {
     const faceClicked = async (e: any, face: FaceData) => {
       if (!identity) {
@@ -175,9 +195,13 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps)
     
       /* Control -- select / deselect single item */
       if (e.ctrlKey) {
+        const cluster = document.querySelector('.Cluster');
         el.classList.toggle('Selected');
-        const selected = [...el.parentElement
+        if (!cluster) {
-          .querySelectorAll('.Selected')]
+          return;
+        }
+
+        const selected = [...cluster.querySelectorAll('.Selected')]
           .map((face: any) => face.getAttribute('data-face-id'));
         setSelected(selected);
         return;
@@ -196,51 +220,57 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps)
     if (identity === undefined) {
       return <></>;
     }
-    return identity.relatedFaces.map(face =>
+    return identity.relatedFaces.map((face: FaceData) => {
-      <div 
+      return (
-        key={face.faceId}
+        <div 
-        style={{
+          key={face.faceId}
-          display: "flex", 
+          style={{
-          alignItems: "center"}}>
+            display: "flex", 
-        <Face
+            justifyContent: 'center',
-          face={face}
+            alignItems: 'center'}}>
-          onFaceClick={faceClicked}
+          <Face
-          title={face.distance}/>
+            face={face}
-      </div>
+            onFaceClick={faceClicked}
-    );
+            title={face.distance}/>
+        </div>
+      );
+    });
   }, [identity, setImage, setSelected]);
 
   const lastNameChanged = (e: any) => {
-    setIdentity(Object.assign(
+    setIdentity({...identity, lastName: e.currentTarget.value }); 
-      {}, 
-      identity, { 
-        lastName: e.currentTarget.value 
-      }
-    ));
   };
   const firstNameChanged = (e: any) => {
-    setIdentity(Object.assign(
+    setIdentity({...identity, firstName: e.currentTarget.value });
-      {},
-      identity, {
-      firstName: e.currentTarget.value
-    }
-    ));
   };
   const middleNameChanged = (e: any) => {
-    setIdentity(Object.assign(
+    setIdentity({...identity, middleName: e.currentTarget.value });
-      {},
-      identity, {
-      middleName: e.currentTarget.value
-    }
-    ));
   };
   const displayNameChanged = (e: any) => {
-    setIdentity(Object.assign(
+    setIdentity({...identity, displayName: e.currentTarget.value });
-      {},
+  };
-      identity, {
+
-      displayName: e.currentTarget.value
+  const updateIdentity = async () => {
+    try {
+      const validFields = [
+        'id', 'displayName', 'firstName', 'lastName', 'middleName'];
+      const filtered: any = Object.assign({}, identity);
+      for (let key in filtered) {
+        if (validFields.indexOf(key) == -1) {
+          delete filtered[key]
+        }
+      }
+      const res = await window.fetch(
+        `${base}/api/v1/identities/${identity.identityId}`, {
+        method: 'PUT',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(filtered)
+      });
+      const data = await res.json();
+      setIdentity({ ...identity });
+    } catch (error) {
+      console.error(error);
     }
-    ));
   };
 
   if (identity === undefined) {
@@ -251,27 +281,27 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps)
 
   return (
     <div className='Cluster'>
-     <div className="Info">
+      <div className="Info">
-      <form>
+        <form className="IdentityForm">
-        <div><label>Last name:<input type="text" 
+          <div>Last name:</div>
-          value={identity.lastName}    
+          <input type="text" 
-          onChange={lastNameChanged}/>
+            value={identity.lastName}    
-        </label></div>
+            onChange={lastNameChanged}/>
-        <div><label>First name:<input type="text" 
+          <div>First name:</div>
-          value={identity.firstName}
+          <input type="text" 
-          onChange={firstNameChanged} />
+            value={identity.firstName}
-        </label></div>
+            onChange={firstNameChanged} />
-        <div><label>Middle name:<input type="text" 
+          <div>Middle name:</div><input type="text" 
-          value={identity.middleName} 
+            value={identity.middleName} 
-          onChange={middleNameChanged} />
+            onChange={middleNameChanged} />
-        </label></div>
+          <div>Display name:</div>
-        <div><label>Display name:<input type="text" 
+          <input type="text" 
-          value={identity.displayName} 
+            value={identity.displayName} 
-          onChange={displayNameChanged} />
+            onChange={displayNameChanged} />
-        </label></div>
+        </form>
-        <div>Faces: {identity.relatedFaces.length}</div>
+        <Button onClick={updateIdentity}>Update</Button> 
-      </form>
       </div>
+      <div>Faces: {identity.relatedFaces.length}</div>
       <div className="Faces">
         { relatedFacesJSX }
       </div>
@@ -316,10 +346,18 @@ const Identities = ({ identities, onFaceClick } : IdentitiesProps) => {
     return identities.map((identity) => {
       const face = identity.relatedFaces[0];
       return (
-        <Face key={face.faceId} 
+        <div
-          face={face}
+          key={face.faceId}
-          onFaceClick={onFaceClick}
+          style={{
-          title={identity.displayName}/>
+            display: "flex",
+            justifyContent: 'center',
+            alignItems: 'center'
+          }}>
+          <Face
+            face={face}
+            onFaceClick={onFaceClick}
+            title={identity.displayName}/>
+        </div>
       );
     });
   }, [ identities, onFaceClick ]);
@@ -342,7 +380,7 @@ const Button = ({ onClick, children }: any) => {
 const App = () => {
   const [identities, setIdentities] = useState<IdentityData[]>([]);
   const { identityId, faceId } = useParams();
-  const [identity, setIdentity] = useState<any>(undefined);
+  const [identity, setIdentity] = useState<IdentityData | undefined>(undefined);
   const [image, setImage] = useState<number>(0);
   const { loading, data } = useApi(
     `${base}/api/v1/identities`
@@ -359,6 +397,31 @@ const App = () => {
     }
   };
 
+  /* If the identity changes, update its entry in the identities list */
+  useEffect(() => {
+    if (!identity || identities.length === 0) {
+      return;
+    }
+    for (let key in identities) {
+      if (identities[key].identityId === identity.identityId) {
+        if (identities[key] !== identity) {
+          /* Inherit all fields from prior identity, while
+           * also keeping the same reference pointer */
+          identities[key] = {
+            ...identity, 
+            relatedFaces: identities[key].relatedFaces
+          };
+          /* relatedFaces is a list of references to identity */
+          identity.relatedFaces.forEach(face => {
+            face.identity = identity;
+          });
+          setIdentities([...identities]);
+        }
+        return;
+      }
+    }
+  }, [identity, setIdentities, identities]);
+
   useEffect(() => {
     if (identityId !== undefined && !isNaN(+identityId)) {
       loadIdentity(+identityId);
@@ -367,6 +430,7 @@ const App = () => {
     if (faceId !== undefined && !isNaN(+faceId)) {
       setImage(+faceId);
     }
+  // eslint-disable-next-line
   }, []);
 
   useEffect(() => {
@@ -380,7 +444,24 @@ const App = () => {
     }
   }, [data]);
 
-  const removeSelected = async () => {
+  const removeFacesFromIdentities = (faceIds: number[]) => {
+    if (!identity) {
+      return;
+    }
+    const pre = identity.relatedFaces.length;
+    /* Remove all relatedFaces which are part of the set of removed
+    * faces */
+    identity.relatedFaces = identity.relatedFaces.filter(
+      (face: FaceData) => faceIds.indexOf(face.faceId) === -1);
+    if (pre !== identity.relatedFaces.length) {
+      setIdentity({ ...identity })
+    }
+  }
+
+  const markSelectedIncorrectIdentity = async () => {
+    if (!identity) {
+      return;
+    }
     try {
       const res = await window.fetch(
         `${base}/api/v1/identities/faces/remove/${identity.identityId}`, {
@@ -390,14 +471,25 @@ const App = () => {
       });
       const data = await res.json();
 
-      const pre = identity.relatedFaces.length;
+      removeFacesFromIdentities(data.faces);
-      /* Remove all relatedFaces which are part of the set of removed
+    } catch (error) {
-       * faces */
+      console.error(error);
-      identity.relatedFaces = identity.relatedFaces.filter(
+    }
-        (face: FaceData) => data.faces.indexOf(face.faceId) === -1);
+  };
-      if (pre !== identity.relatedFaces.length) {
+
-        setIdentity({...identity})
+  const markSelectedNotFace = async () => {
-      }
+    try {
+      const res = await window.fetch(
+        `${base}/api/v1/faces`, {
+        method: 'PUT',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          action: 'not-a-face',
+          faces: selected 
+        })
+      });
+      const data = await res.json();
+      removeFacesFromIdentities(data);
     } catch (error) {
       console.error(error);
     }
@@ -409,17 +501,14 @@ const App = () => {
       return;
     }
     const identityId = face.identity.identityId;
-    console.log(face.identity);
+    const faceId = face.faceId;
-    const identitiesEl = document.querySelector('.Identities');
+    console.log(`onFaceClick`, { faceId, identityId});
-    if (!identitiesEl) {
+    const faces = [
-      return;
+      ...document.querySelectorAll(`.Identities [data-identity-id="${identityId}"]`),
-    }
+      ...document.querySelectorAll(`.Cluster [data-face-id="${faceId}"]`)];
-    const faceIdentity = identitiesEl.querySelector(
+    faces.forEach((el: any) => {
-      `[data-identity-id="${identityId}"]`);
+      el.scrollIntoView();
-    if (!faceIdentity) {
+    });
-      return;
-    }
-    faceIdentity.scrollIntoView()
   };
 
   const identitiesOnFaceClick = (e: any, face: FaceData) => {
@@ -434,18 +523,21 @@ const App = () => {
           autoSaveId="persistence" direction="horizontal">
           <Panel defaultSize={50} className="ClusterEditor">
             {loading && <div style={{ margin: '1rem' }}>Loading...</div>}
-            {!loading && identity !== 0 && 
+            {!loading && identity !== undefined && 
               <Cluster {...{ 
                 identity, 
                 setIdentity, 
                 setImage,  
                 setSelected 
               }} />}
-            {!loading && identity === 0 && <div className="Cluster">
+            {!loading && identity === undefined && <div className="Cluster">
               Select identity to edit
             </div>}
             <div className="Actions">
-              { selected.length !== 0 && <Button onClick={removeSelected}>Remove</Button> }
+              { selected.length !== 0 && <>
+                <Button onClick={markSelectedIncorrectIdentity}>Remove</Button> 
+                <Button onClick={markSelectedNotFace}>Not a face</Button> 
+              </>}
             </div>
           </Panel>
           <PanelResizeHandle className="Resizer"/>
@@ -453,9 +545,12 @@ const App = () => {
             {image === 0 && <div style={{ margin: '1rem' }}>Select image to view</div>}
             {image !== 0 && <Photo onFaceClick={onFaceClick} photoId={image}/> }
           </Panel>
-        </PanelGroup>
+          <PanelResizeHandle className="Resizer" />
-        { !loading && <Identities 
+          <Panel defaultSize={8.5} minSize={8.5} className="IdentitiesList">
-          {... { onFaceClick: identitiesOnFaceClick, identities }}/> }
+            { !loading && <Identities 
+              {... { onFaceClick: identitiesOnFaceClick, identities }}/> }
+          </Panel>
+          </PanelGroup>
       </div>
     </div>
   );

ketrface/cluster.py

@@ -27,19 +27,6 @@ if html_base == "/":
 MAX_CLUSTER_DISTANCE = 0.14       # Used to merge clusters
 MAX_DISTANCE_FROM_CENTROID = 0.14 # Used to prune outliers
 
-# TODO
-# Switch to using DBSCAN
-#
-# Thoughts for determining number of clusters to try and target...
-#
-# Augment DBSCAN to rule out identity matching for the same face
-# appearing more than once in a photo
-#
-# NOTE: This means twins or reflections won't both identify in the
-# same photo -- those faces would then identify as a second face pairing
-# which could merge with a cluster, but can not be used to match
-
-
 def gen_html(identities):
   for identity in identities:
 
@@ -79,41 +66,6 @@ def update_cluster_averages(identities):
         average, average)))
   return identities
 
-def load_faces(db_path = db_path):
-  print(f'Connecting to database: {db_path}')
-  conn = create_connection(db_path)
-  faces = []
-  with conn:
-    print('Querying faces')
-    cur = conn.cursor()
-    res = cur.execute('''
-      SELECT faces.id,facedescriptors.descriptors,faces.faceConfidence,faces.photoId,faces.focus
-      FROM faces
-      INNER JOIN photos ON (photos.duplicate == 0 OR photos.duplicate IS NULL)
-      JOIN facedescriptors ON (faces.descriptorId=facedescriptors.id)
-      WHERE faces.identityId IS null AND faces.faceConfidence>0.99
-        AND faces.photoId=photos.id
-      ''')
-    for row in res.fetchall():
-      id, descriptors, confidence, photoId, focus = row
-      if focus is None:
-        focus = 100 # Assume full focus if focus not set
-      face = { 
-        'id': id,
-        'type': 'face',
-        'confidence': confidence,
-        'distance': 0,
-        'photoId': photoId,
-        'descriptors': np.frombuffer(descriptors),
-        'cluster': Undefined,
-        'focus': focus
-      }
-      face['faces'] = [ face ]
-      face['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
-        face['descriptors'], face['descriptors'])))
-      faces.append(face)
-  return faces
-
 def update_distances(identities, 
     prune = False, 
     maxDistance = MAX_DISTANCE_FROM_CENTROID):
@@ -169,7 +121,7 @@ def build_straglers(faces):
   return noise + undefined
 
 print('Loading faces from database')
-faces = load_faces()
+faces = load_faces(db_path = db_path)
 
 minPts = max(len(faces) / 500, 5)
 eps = 0.185
@@ -267,41 +219,6 @@ redirect_on(os.path.join(html_path, 'auto-clusters.html'))
 gen_html(reduced)
 redirect_off()
 
-def create_identity(conn, identity):
-  """
-  Create a new identity in the identities table
-  :param conn:
-  :param identity:
-  :return: identity id
-  """
-  sql = '''
-  INSERT INTO identities(descriptors,displayName)
-  VALUES(?,?)
-  '''
-  cur = conn.cursor()
-  cur.execute(sql, (
-    np.array(identity['descriptors']),
-    f'cluster-{identity["id"]}'
-  ))
-  conn.commit()
-  return cur.lastrowid
-
-def update_face_identity(conn, faceId, identityId = None):
-  """
-  Update the identity associated with this face
-  :param conn:
-  :param faceId:
-  :param identityId:
-  :return: None
-  """
-  sql = '''
-  UPDATE faces SET identityId=? WHERE id=?
-  '''
-  cur = conn.cursor()
-  cur.execute(sql, (identityId, faceId))
-  conn.commit()
-  return None
-
 print(f'Connecting to database: {db_path}')
 conn = create_connection(db_path)
 with conn:

ketrface/ketrface/db.py

@@ -76,3 +76,83 @@ def update_face_count(conn, photoId, faces):
   cur.execute(sql, (faces, photoId))
   conn.commit()
   return None
+
+
+import sys
+import json
+import os
+import piexif
+import sqlite3
+from sqlite3 import Error
+from PIL import Image
+import numpy as np
+
+def load_faces(db_path ):
+  print(f'Connecting to database: {db_path}')
+  conn = create_connection(db_path)
+  faces = []
+  with conn:
+    print('Querying faces')
+    cur = conn.cursor()
+    res = cur.execute('''
+      SELECT faces.id,facedescriptors.descriptors,faces.faceConfidence,faces.photoId,faces.focus
+      FROM faces
+      INNER JOIN photos ON (photos.duplicate == 0 OR photos.duplicate IS NULL)
+      JOIN facedescriptors ON (faces.descriptorId=facedescriptors.id)
+      WHERE faces.identityId IS null AND faces.faceConfidence>0.99
+        AND faces.photoId=photos.id
+      ''')
+    for row in res.fetchall():
+      id, descriptors, confidence, photoId, focus = row
+      if focus is None:
+        focus = 100 # Assume full focus if focus not set
+      face = { 
+        'id': id,
+        'type': 'face',
+        'confidence': confidence,
+        'distance': 0,
+        'photoId': photoId,
+        'descriptors': np.frombuffer(descriptors),
+        'cluster': 0, # Undefined from dbscan.py
+        'focus': focus
+      }
+      face['faces'] = [ face ]
+      face['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+        face['descriptors'], face['descriptors'])))
+      faces.append(face)
+  return faces
+
+def create_identity(conn, identity):
+  """
+  Create a new identity in the identities table
+  :param conn:
+  :param identity:
+  :return: identity id
+  """
+  sql = '''
+  INSERT INTO identities(descriptors,displayName)
+  VALUES(?,?)
+  '''
+  cur = conn.cursor()
+  cur.execute(sql, (
+    np.array(identity['descriptors']),
+    f'cluster-{identity["id"]}'
+  ))
+  conn.commit()
+  return cur.lastrowid
+
+def update_face_identity(conn, faceId, identityId = None):
+  """
+  Update the identity associated with this face
+  :param conn:
+  :param faceId:
+  :param identityId:
+  :return: None
+  """
+  sql = '''
+  UPDATE faces SET identityId=? WHERE id=?
+  '''
+  cur = conn.cursor()
+  cur.execute(sql, (identityId, faceId))
+  conn.commit()
+  return None

ketrface/split.py

@@ -0,0 +1,255 @@
+import sys
+import json
+import os
+import piexif
+import sqlite3
+from sqlite3 import Error
+from PIL import Image
+import numpy as np
+
+import functools
+
+from ketrface.util import *
+from ketrface.dbscan import *
+from ketrface.db import *
+from ketrface.config import *
+
+MAX_DISTANCE_FROM_CENTROID = 0.14 # Used to prune outliers
+
+config = read_config()
+
+html_path = merge_config_path(config['path'], 'frontend')
+pictures_path = merge_config_path(config['path'], config['picturesPath'])
+faces_path = merge_config_path(config['path'], config['facesPath'])
+db_path = merge_config_path(config['path'], config["db"]["photos"]["host"])
+html_base = config['basePath']
+if html_base == "/":
+  html_base = "."
+
+def update_cluster_averages(identities):
+  for identity in identities:
+    average = []
+    for face in identity['faces']:
+      if len(average) == 0:
+        average = face['descriptors']
+      else:
+        average = np.add(average, face['descriptors'])
+    average = np.divide(average, len(identity['faces']))
+    identity['descriptors'] = average
+    identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+        average, average)))
+  return identities
+
+
+def sort_identities(identities):
+  identities.sort(reverse = True, key = lambda x: len(x['faces']))
+  for identity in identities:
+    identity['faces'].sort(reverse = False, key = lambda x: x['distance'])
+
+def cluster_sort(A, B):
+  diff = A['cluster'] - B['cluster']
+  if diff > 0:
+    return 1
+  elif diff < 0:
+    return -1
+  diff = A['confidence'] - B['confidence']
+  if diff > 0:
+    return 1
+  elif diff < 0:
+    return -1
+  return 0
+
+def load_identities(db_path):
+  conn = create_connection(db_path)
+  identities = []
+  with conn:
+    cur = conn.cursor()
+    res = cur.execute('''
+      SELECT
+        identities.id as identityId,
+        identities.displayName as displayName,
+        identities.descriptors as descriptors,
+        COUNT(faces.id) AS faceCount
+      FROM identities 
+      JOIN faces ON faces.identityId=identities.id
+      GROUP BY identities.id
+    ''')
+    for row in res.fetchall():
+      identityId, displayName, descriptors, faceCount = row
+      identity = {
+        'identityId': identityId,
+        'displayName': displayName,
+        'descriptors': np.frombuffer(descriptors),
+        'faceCount': faceCount,
+        'updated': False
+      }
+      # Pre-bake computations for cosine distance
+      identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+        identity['descriptors'], identity['descriptors'])))
+      identities.append(identity)
+  return identities
+
+def find_identity(identities, identityId):
+  for element in identities:
+    if element['identityId'] == identityId:
+      return element
+  raise Exception(f'Identity {identityId} missing')
+  
+def load_doppelganger_photos(db_path):
+  conn = create_connection(db_path)
+  photos = {}
+  with conn:
+    cur = conn.cursor()
+    res = cur.execute('''
+      SELECT 
+        f1.identityId AS identityId,
+        photos.id AS photoId,
+        f1.id AS f1_id,
+        f2.id AS f2_id,
+        f1_descriptors.descriptors AS f1_descriptors,
+        f2_descriptors.descriptors AS f2_descriptors
+      FROM faces AS f1 
+      INNER JOIN faces AS f2 ON (
+        f2.identityId=f1.identityId AND f1.photoId=f2.photoId and f1_id!=f2_id)
+      INNER JOIN photos ON (
+        photos.duplicate == 0 OR photos.duplicate IS NULL)
+      INNER JOIN facedescriptors AS f1_descriptors ON (
+        f1.descriptorId=f1_descriptors.id)
+      INNER JOIN facedescriptors AS f2_descriptors ON (
+        f2.descriptorId=f2_descriptors.id)
+      WHERE f1.identityId IS NOT NULL AND f1.photoId=photos.id
+      ORDER BY photos.id,f1.identityId
+      ''')
+    for row in res.fetchall():
+      identityId, photoId, f1_id, f2_id,f1_descriptors, f2_descriptors = row
+      face1 = { 
+        'id': f1_id,
+        'type': 'face',
+        'distance': 0,
+        'descriptors': np.frombuffer(f1_descriptors),
+        'cluster': identityId, # Undefined from dbscan.py
+      }
+      face1['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+        face1['descriptors'], face1['descriptors'])))
+      
+      face2 = { 
+        'id': f2_id,
+        'type': 'face',
+        'distance': 0,
+        'descriptors': np.frombuffer(f2_descriptors),
+        'cluster': identityId, # Undefined from dbscan.py
+      }
+      face2['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+        face2['descriptors'], face2['descriptors'])))
+      
+      if photoId not in photos:
+        photos[photoId] = {
+          'photoId': photoId,
+          'dopplegangers': {}
+        }
+      if identityId not in photos[photoId]['dopplegangers']:
+        photos[photoId]['dopplegangers'][identityId] = {
+          'identity': None,
+          'faces': []
+        }
+
+      faceList = photos[photoId]['dopplegangers'][identityId]['faces']
+      for face in [ face1, face2 ]:      
+        match = False
+        for key in faceList:
+          if face['id'] == key['id']:
+            match = True
+            break
+        if not match:
+          faceList.append(face)
+
+    return photos
+
+def remove_face_from_identity(identity, face):
+  identity['updated'] = True
+  average = identity['descriptors']
+  average = np.dot(average, identity['faceCount'])
+  average = np.subtract(average, face['descriptors'])
+  identity['faceCount'] -= 1
+  average = np.divide(average, identity['faceCount'])
+  identity['descriptors'] = average
+  identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+    average, average)))
+
+  face['identity'] = None
+  face['identityId'] = -1
+  face['distance'] = 0
+
+print('Loading identities from database...')
+identities = load_identities(db_path)
+print(f'{len(identities)} identities loaded.')
+print('Loading dopplegangers from database...')
+photos = load_doppelganger_photos(db_path)
+print(f'{len(photos)} photos with dopplegangers loaded.')
+print('Binding dopplegangers to identities...')
+
+face_updates = []
+identity_updates = []
+for photoId in photos:
+  photo = photos[photoId]
+  print(f'Processing photo {photoId}...')
+  for identityId in photo['dopplegangers']:
+    if photo['dopplegangers'][identityId]['identity'] == None:
+      photo['dopplegangers'][identityId]['identity'] = find_identity(identities, identityId)
+    faces = photo['dopplegangers'][identityId]['faces']
+    identity = photo['dopplegangers'][identityId]['identity']
+  
+    for face in faces:
+      face['identity'] = identity
+      face['distance'] = findCosineDistanceBaked(face, identity)
+    faces.sort(reverse = False, key = lambda x: x['distance'])
+   
+    # First face closest to identity -- it stays with the photo
+    faces = faces[1:]
+
+    # 
+    for i, face in enumerate(faces):
+      remove_face_from_identity(identity, face)
+      identity_updates.append(identity)
+      min = None
+      for j, potential in enumerate(identities):
+        if potential == identity:
+          continue
+        distance = findCosineDistanceBaked(face, potential)
+        if distance > MAX_DISTANCE_FROM_CENTROID:
+          continue
+        if min == None or distance < min:
+          face["distance"] = distance
+          face['identity'] = potential
+          face['identityId'] = potential['identityId']
+
+      face_updates.append(face)
+
+      if face['identity'] != None:
+        print(f'  {i+1}: {face["id"]} moves from {identity["displayName"]} to:')
+        print(f'    {face["identity"]["displayName"]} with distance {face["distance"]}')
+      else:
+        print(f'  {i+1}: {face["id"]} needs a new Identity.')
+  
+
+conn = create_connection(db_path)
+with conn:
+  cur = conn.cursor()
+  for face in face_updates:
+    print(f'Updating face {face["id"]} in DB')
+    if face['identity'] == None:
+      sql = '''
+        UPDATE faces SET identityId=NULL WHERE id=?
+      '''
+      values=(face["id"], )
+    else:
+      sql = '''
+      UPDATE faces SET identityId=? WHERE id=?
+      '''
+      values=(
+        face["identityId"], 
+        face["id"]
+      )
+    cur.execute(sql, values)
+  conn.commit()
+

server/db/photos.js

@@ -172,12 +172,13 @@ function init() {
           key: 'id',
         }
       },
-
+      classifiedBy: {
-      expertAssignment: {
+        type: Sequelize.DataTypes.ENUM(
-        type: Sequelize.BOOLEAN,
+          'machine', 
-        defaultValue: false
+          'human', 
+          'not-a-face'), /* implies "human"; identityId=NULL */
+        defaultValue: 'machine',
       },
-      
       lastComparedId: {
         type: Sequelize.INTEGER,
         allowNull: true,

server/routes/faces.js

@@ -14,12 +14,52 @@ require("../db/photos").then(function(db) {
 const router = express.Router();
 const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
 
-router.put("/:id", function(req, res/*, next*/) {
+router.put("/:id?", async (req, res/*, next*/) => {
+  console.log(`PUT ${req.url}`);
+
   if (!req.user.maintainer) {
-    return res.status(401).send("Unauthorized to modify photos.");
+    return res.status(401).json({ message: "Unauthorized to modify photos."});
   }
 
-  return res.status(400).send("Invalid request");
+  let { id } = req.params, faces = [];
+  if (id && isNaN(+id)) {
+    return res.status(400).json({message: `Invalid id ${id}`});
+  } else {
+    if (id) {
+      faces = [ +id ];
+    } else {
+      faces = req.body.faces;
+    }
+  }
+  if (faces.length === 0) {
+    return res.status(400).json({message: `No faces supplied.`});
+  }
+  const { action } = req.body; 
+  console.log(`${action}: ${faces}`);
+  switch (action) {
+  case 'not-a-face':
+    await photoDB.sequelize.query(
+      `UPDATE faces SET classifiedBy='not-a-face',identityId=NULL ` +
+      `WHERE id IN (:faces)`, {
+        replacements: { faces }
+      }
+    );
+    /*
+    faces = await photoDB.sequelize.query(
+      'SELECT * FROM faces WHERE id IN (:faces)', {
+        replacements: { faces },
+        type: photoDB.Sequelize.QueryTypes.SELECT,
+        raw: true
+      }
+    );
+    faces.forEach(face => {
+      face.faceId = face.id;
+      delete face.id;
+    });
+    */
+    return res.status(200).json(faces);
+  }
+  return res.status(400).json({ message: "Invalid request" });
 });
 
 router.delete("/:id", function(req, res/*, next*/) {

server/routes/identities.js

@@ -11,6 +11,58 @@ require("../db/photos").then(function(db) {
 
 const router = express.Router();
 
+router.put('/:id', async (req, res) => {
+  console.log(`PUT ${req.url}`)
+  if (!req.user.maintainer) {
+    console.warn(`${req.user.name} attempted to modify photos.`);
+    return res.status(401).send({ message: "Unauthorized to modify photos." });
+  }
+
+  const { id } = req.params;
+  if (!id || isNaN(+id)) {
+    return res.status(400).send({ message: `Invalid identity id ${id}` });
+  }
+
+  const {
+    displayName,
+    firstName,
+    lastName,
+    middleName
+  } = req.body;
+
+  if (displayName === undefined
+    || firstName === undefined 
+    || lastName === undefined
+    || middleName === undefined) {
+    return res.status(400).send({ message: `Missing fields` });
+  }
+
+  await photoDB.sequelize.query(
+    'UPDATE identities ' +
+    'SET ' +
+      'displayName=:displayName, ' +
+      'firstName=:firstName, ' +
+      'lastName=:lastName, ' +
+      'middleName=:middleName ' +
+    'WHERE id=:id', {
+      replacements: {
+        displayName,
+        firstName,
+        lastName,
+        middleName,
+        id
+      }
+    }
+  );
+
+  return res.status(200).json({
+    displayName,
+    firstName,
+    lastName,
+    middleName,
+    id
+  });
+});
 
 router.put("/faces/remove/:id", (req, res) => {
   console.log(`PUT ${req.url}`)