Improved UX

Signed-off-by: James P. Ketrenos <james.p.ketrenos@intel.com>

client/package-lock.json

@@ -17,6 +17,7 @@
         "@types/react-dom": "^18.0.10",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
+        "react-resizable-panels": "^0.0.34",
         "react-scripts": "5.0.1",
         "typescript": "^4.9.4",
         "web-vitals": "^2.1.4"
@@ -14226,6 +14227,15 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/react-resizable-panels": {
+      "version": "0.0.34",
+      "resolved": "https://registry.npmjs.org/react-resizable-panels/-/react-resizable-panels-0.0.34.tgz",
+      "integrity": "sha512-GGT69jbCiK5Fmw7p9mopb+quX63g+OA235bSHtj8TD3O+wsFNgrg9j5TaRI6auP1J10SBmR0OpJ7tX3K7MFxeg==",
+      "peerDependencies": {
+        "react": "^16.14.0 || ^17.0.0 || ^18.0.0",
+        "react-dom": "^16.14.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
     "node_modules/react-scripts": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/react-scripts/-/react-scripts-5.0.1.tgz",

client/package.json

@@ -13,6 +13,7 @@
     "@types/react-dom": "^18.0.10",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
+    "react-resizable-panels": "^0.0.34",
     "react-scripts": "5.0.1",
     "typescript": "^4.9.4",
     "web-vitals": "^2.1.4"

client/src/App.css

@@ -19,6 +19,21 @@ div {
   height: 100%;
 }
 
+.Resizer {
+  width: 0.5rem;
+  border: 1px solid black;
+}
+
+.Explorer {
+  display: flex;
+  flex-direction: row;
+  justify-self: stretch;
+  align-self: stretch;
+  height: 100%;
+  width: auto !important;
+  flex-grow: 1;
+}
+
 .Identities {
   display: flex;
   overflow-y: scroll;
@@ -71,11 +86,12 @@ div {
 }
 
 .Cluster {
+  user-select: none;
   display: flex;
   flex-direction: column;
   overflow-y: scroll;
-  flex-grow: 1;
   padding: 0.5rem;
+  height: 100%;
 }
 
 .Cluster .Face.Selected {

client/src/App.tsx

@@ -1,8 +1,36 @@
 
 import React, { useState, useMemo, useEffect } from 'react';
 import { useApi } from './useApi';
+import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels";
 import './App.css';
 
+const Photo = ({ photoId }: any) => {
+  const [image, setImage] = useState<any>(undefined);
+
+  useEffect(() => {
+    if (photoId === 0) {
+      return;
+    }
+    const fetchImageData = async (image: number) => {
+      console.log(`Loading photo ${image}`);
+      const res = await window.fetch(`../api/v1/photos/${image}`);
+      const data = await res.json();
+      setImage(data[0]);
+    };
+
+    fetchImageData(photoId);
+  }, [photoId, setImage]);
+
+  if (image === undefined) {
+    return <></>
+  }
+
+  return (<div className="Image"
+    style={{
+      background: `url(${image.path}thumbs/scaled/${image.filename})`
+    }}/>);
+};
+
 const Face = ({ faceId, onClick, title }: any) => {
   const idPath = String(faceId % 100).padStart(2, '0');
   return (
@@ -19,10 +47,11 @@ const Face = ({ faceId, onClick, title }: any) => {
 };
 
 type ClusterProps = {
-  id: number
+  id: number,
+  setImage(image: number): void
 };
 
-const Cluster = ({ id }: ClusterProps) => {
+const Cluster = ({ id, setImage }: ClusterProps) => {
   const [identity, setIdentity] = useState<Identity | undefined>(undefined);
   const { loading, data } = useApi(
     `../api/v1/identities/${id}`
@@ -38,15 +67,22 @@ const Cluster = ({ id }: ClusterProps) => {
     }
   }, [data]);
 
-  
-
   const relatedFacesJSX = useMemo(() => {
-    const faceClicked = (e: any, id: any) => {
+    const faceClicked = async (e: any, id: any) => {
       if (!identity) {
         return;
       }
       const el = e.currentTarget;
       const face = identity.relatedFaces.find(item => item.faceId === id);
+      if (!face) {
+        return;
+      }
+      if (e.shiftKey) {
+        e.stopPropagation();
+        e.preventDefault();
+        setImage(face.photoId);
+        return;
+      }
       el.classList.toggle('Selected');
       console.log(face);
     }
@@ -59,7 +95,7 @@ const Cluster = ({ id }: ClusterProps) => {
         onClick={faceClicked}
         title={face.distance}/>
     );
-  }, [identity]);
+  }, [identity, setImage]);
 
   const lastNameChanged = (e: any) => {
     setIdentity(Object.assign(
@@ -138,6 +174,7 @@ const Cluster = ({ id }: ClusterProps) => {
 
 type FaceData = {
   faceId: number,
+  photoId: number,
   lastName: string,
   firstName: string,
   middleName: string,
@@ -190,6 +227,7 @@ const Identities = ({ identities, setIdentity } : IdentitiesProps) => {
 const App = () => {
   const [identities, setIdentities] = useState<Identity[]>([]);
   const [identity, setIdentity] = useState<number>(0);
+  const [image, setImage] = useState<number>(0);
   const { loading, data } = useApi(
     '../api/v1/identities'
   );
@@ -203,14 +241,22 @@ const App = () => {
   return (
     <div className="App">
       <div className="Worksheet">
-        { loading && <div style={{margin:'1rem'}}>Loading...</div> }
+        <PanelGroup className="Explorer" 
-        { !loading && identity !== 0 && <Cluster id={identity} />}
+          autoSaveId="persistence" direction="horizontal">
-        { !loading && identity === 0 && <div className="Cluster">
+          <Panel defaultSize={50}>
-          Select identity to edit
+            {loading && <div style={{ margin: '1rem' }}>Loading...</div>}
-        </div> }
+            {!loading && identity !== 0 && <Cluster id={identity} {...{ setImage, }} />}
-        { !loading && <>
+            {!loading && identity === 0 && <div className="Cluster">
-          <Identities {... {identities, setIdentity }}/>
+              Select identity to edit
-        </> }
+            </div>}
+          </Panel>
+          <PanelResizeHandle className="Resizer"/>
+          <Panel>
+            {image === 0 && <div style={{ margin: '1rem' }}>Select image to view</div>}
+            {image !== 0 && <Photo photoId={image}/> }
+          </Panel>
+        </PanelGroup>
+        { !loading && <Identities {... {identities, setIdentity }}/> }
       </div>
     </div>
   );

ketrface/cluster.py

@@ -72,6 +72,8 @@ def update_cluster_averages(identities):
         average = np.add(average, face['descriptors'])
     average = np.divide(average, len(identity['faces']))
     identity['descriptors'] = average
+    identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+        average, average)))
   return identities
 
 def load_faces(db_path = db_path):
@@ -102,6 +104,8 @@ def load_faces(db_path = db_path):
         'focus': focus
       }
       face['faces'] = [ face ]
+      face['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+        face['descriptors'], face['descriptors'])))
       faces.append(face)
   return faces
 
@@ -155,9 +159,10 @@ def build_straglers(faces):
 
 print('Loading faces from database')
 faces = load_faces()
-print(f'{len(faces)} faces loaded')
+minPts = len(faces) / 100
-print('Scanning for clusters')
+eps = 0.2
-identities = DBSCAN(faces) # process_faces(faces)
+print(f'Scanning {len(faces)} faces for clusters (minPts: {minPts}, eps: {eps})')
+identities = DBSCAN(faces, minPts = minPts, eps = eps)
 print(f'{len(identities)} clusters grouped')
 
 MAX_CLUSTER_DISTANCE = 0.15  # Used to merge clusters
@@ -166,37 +171,41 @@ MAX_EPOCH_DISTANCE = 0.14    # Used to prune outliers
 # Compute average center for all clusters
 identities = update_cluster_averages(identities)
 
-removed = -1
+if False:
-epoch = 1
+  removed = -1
-# Filter each cluster removing any face that is > cluster_max_distance
+  epoch = 1
-# from the average center point of the cluster
+  # Filter each cluster removing any face that is > cluster_max_distance
-while removed != 0:
+  # from the average center point of the cluster
-  print(f'Epoch {epoch}...')
+  while removed != 0:
-  epoch += 1
+    print(f'Epoch {epoch}...')
-  removed = update_distances(identities, prune = True)
+    epoch += 1
-  if removed > 0:
+    removed = update_distances(identities, prune = True)
-    print(f'Excluded {removed} faces this epoch')
+    if removed > 0:
+      print(f'Excluded {removed} faces this epoch')
 
 print(f'{len(identities)} identities seeded.')
+reduced = identities
 
-# Cluster the clusters...
+if False:
-print('Reducing clusters via DBSCAN')
+  # Cluster the clusters...
-reduced = DBSCAN(identities, eps = MAX_CLUSTER_DISTANCE, minPts = 2)
+  print('Reducing clusters via DBSCAN')
-if len(reduced) == 0:
+  reduced = DBSCAN(identities, eps = MAX_CLUSTER_DISTANCE, minPts = 3)
-  reduced = identities
+  if len(reduced) == 0:
-# For each cluster, merge the lists of faces referenced in the cluster's
+    reduced = identities
-# "faces" field, which is pointing to clusters (and not actual faces)
+  # For each cluster, merge the lists of faces referenced in the cluster's
-for cluster in reduced:
+  # "faces" field, which is pointing to clusters (and not actual faces)
-  merged = []
+  for cluster in reduced:
-  for identity in cluster['faces']:
+    merged = []
-    merged = merged + identity['faces']
+    for identity in cluster['faces']:
-  cluster['faces'] = merged
+      merged = merged + identity['faces']
+    cluster['faces'] = merged
 
-# Creating a set containing those faces which have not been bound
+if False:
-# to an identity to recluster them in isolation from the rest of
+  # Creating a set containing those faces which have not been bound
-# the faces
+  # to an identity to recluster them in isolation from the rest of
-straglers = build_straglers(faces)
+  # the faces
-reduced = reduced + DBSCAN(straglers)
+  straglers = build_straglers(faces)
+  reduced = reduced + DBSCAN(straglers)
 
 # Build a final cluster with all remaining uncategorized faces
 if False:
@@ -225,20 +234,21 @@ update_distances(reduced)
 
 sort_identities(reduced)
 
-# This generates a set of differences between clusters and makes
+if False:
-# a recommendation to merge clusters (outside of DBSCAN)
+  # This generates a set of differences between clusters and makes
-#
+  # a recommendation to merge clusters (outside of DBSCAN)
-# Worth testing on larger data set
+  #
-for i, A in enumerate(reduced):
+  # Worth testing on larger data set
-  for k, B in enumerate(reduced):
+  for i, A in enumerate(reduced):
-    if k < i:
+    for k, B in enumerate(reduced):
-      continue
+      if k < i:
-    if A == B:
+        continue
-      continue
+      if A == B:
-    distance = findCosineDistance(A['descriptors'], B['descriptors'])
+        continue
-    if distance < MAX_CLUSTER_DISTANCE:
+      distance = findCosineDistanceBaked(A, B)
-      distance = "{:0.4f}".format(distance)
+      if distance < MAX_CLUSTER_DISTANCE:
-      print(f'{A["id"]} to {B["id"]} = {distance}: MERGE')
+        distance = "{:0.4f}".format(distance)
+        print(f'{A["id"]} to {B["id"]} = {distance}: MERGE')
 
 print('Writing to "auto-clusters.html"')
 redirect_on(os.path.join(html_path, 'auto-clusters.html'))

ketrface/ketrface/dbscan.py

@@ -11,10 +11,13 @@ Noise = -2
 # Union of two lists of dicts, adding unique elements of B to
 # end of A
 def Union(A, B):
-  for key in B:
+# 5.012 of 100s sample
-    if key not in A:
+  return A + [x for x in B if x not in A]
-      A.append(key)
+# 5.039 of 100s sample
-  return A    
+#  for key in B:
+#    if key not in A:
+#      A.append(key)
+#  return A
 
 # https://en.wikipedia.org/wiki/DBSCAN
 #
@@ -24,19 +27,24 @@ def DBSCAN(points, eps = MAX_DISTANCE, minPts = MIN_PTS, verbose = True):
   total = len(points)
   last = 0
   start = time.time()
+
+  # NOTE: A point (P) is only scanned once by RangeQuery for cluster
+  # inclusion. The internal loop does not need to rescan those points
+  # as they would have already been in a cluster if minPts is reached.
   for i, P in enumerate(points):
+
     if verbose == True:
       new_perc = int(100 * (i+1) / total)
       now = time.time()
       if new_perc != perc or now - last > 5:
         perc = new_perc
-        print(f'Clustering points {perc}% ({i}/{total} processed) complete with {len(clusters)} identities ({now - start}s).')
         last = now
+        print(f'Clustering points {perc}% ({i}/{total} processed) complete with {len(clusters)} identities ({int(now - start)}s).')
       
     if P['cluster'] != Undefined:     # Previously processed in inner loop
       continue
 
-    N = RangeQuery(points, P, eps)    # Find neighbors
+    N = RangeQuery(points[i:], P, eps)# Find neighbors
     if len(N) < minPts:               # Density check
       P['cluster'] = Noise            # Label as Noise
       continue
@@ -49,8 +57,7 @@ def DBSCAN(points, eps = MAX_DISTANCE, minPts = MIN_PTS, verbose = True):
     clusters.append(C)
 
     P['cluster'] = C                  # Label initial point
-    S = N                             # Neighbors to expand (exclude P)
+    S = N                             # Neighbors to expand (excludes P)
-    S.remove(P)
 
     sub_perc = -1
     sub_last = 0
@@ -58,13 +65,11 @@ def DBSCAN(points, eps = MAX_DISTANCE, minPts = MIN_PTS, verbose = True):
     for j, Q in enumerate(S):        # Process every seed point
 
       if verbose == True:
-        sub_total = len(S)
-        sub_new_perc = int(100 * (j+1) / sub_total)
         sub_now = time.time()
-        if sub_new_perc != sub_perc or sub_now - sub_last > 5:
+        if sub_now - sub_last > 5:
-          sub_perc = sub_new_perc
+          sub_total = len(S)
-          print(f'... points {sub_perc}% ({j}/{sub_total} processed [{perc}% total]) complete with {len(clusters)} identities ({now - start}s).')
           sub_last = sub_now
+          print(f'... points {j}/{sub_total} processed [{perc}% total]). {len(C["faces"])} forming: {len(clusters)} identities ({int(sub_now - start)}s).')
 
       if Q['cluster'] == Noise:       # Change Noise to border point
         Q['cluster'] = C
@@ -76,7 +81,7 @@ def DBSCAN(points, eps = MAX_DISTANCE, minPts = MIN_PTS, verbose = True):
       Q['cluster'] = C                # Label neighbor
       C['faces'].append(Q)
       
-      N = RangeQuery(points, Q, eps)  # Find neighbors
+      N = RangeQuery(points[i:], Q, eps)  # Find neighbors
       if len(N) >= minPts:            # Density check (if Q is a core point)
         S = Union(S, N)               # Add new neighbors to seed set
   return clusters
@@ -86,12 +91,9 @@ def RangeQuery(points, Q, eps):
   for P in points:                    # Scan all points in the database
     if P == Q:
       continue
-    if P in neighbors:
+    distance = findCoseinDistanceBaked(# Compute distance
-      continue       
+      Q, P) 
-    distance = findCosineDistance(    # Compute distance and check epsilon
+    if distance <= eps:               # Check epsilon
-      Q['descriptors'], 
+      neighbors.append(P)             # Add to result  
-      P['descriptors'])
-    if distance <= eps:
-      neighbors += [ P ]              # Add to result  
   return neighbors
 

ketrface/ketrface/util.py

@@ -47,11 +47,15 @@ class NpEncoder(json.JSONEncoder):
     if isinstance(obj, np.ndarray):
       return obj.tolist()
 
+def findCoseinDistanceBaked(src, dst):
+  a = np.matmul(np.transpose(src['descriptors']), dst['descriptors'])
+  return 1 - (a / (src['sqrtsummul'] * dst['sqrtsummul']))
+
 def findCosineDistance(source_representation, test_representation):
-  if type(source_representation) == list:
+#  if type(source_representation) == list:
-    source_representation = np.array(source_representation)
+#    source_representation = np.array(source_representation)
-  if type(test_representation) == list:
+#  if type(test_representation) == list:
-    test_representation = np.array(test_representation)
+#    test_representation = np.array(test_representation)
   a = np.matmul(np.transpose(source_representation), test_representation)
   b = np.sum(np.multiply(source_representation, source_representation))
   c = np.sum(np.multiply(test_representation, test_representation))

server/routes/photos.js

@@ -1083,6 +1083,31 @@ console.log("Trying path as: " + path);
   });
 });
 
+router.get("/:id", async (req, res) => {
+  const id = parseInt(req.params.id);
+  try {
+    const results = await photoDB.sequelize.query(
+      `
+      SELECT photos.*,albums.path AS path,
+        faces.identityId,faces.top,faces.left,faces.right,faces.bottom 
+      FROM photos
+      INNER JOIN albums ON albums.id=photos.albumId 
+      INNER JOIN faces ON faces.photoId=photos.id 
+      WHERE photos.id=:id
+      `, {
+        replacements: { id }
+      }
+    );
+    if (results.length === 0) {
+      return res.status(404);
+    }
+    return res.status(200).json(results[0]);
+  } catch (error) {
+    console.error(error);
+    return res.status(404).json({message: `Error connecting to DB for ${id}.`})
+  }
+});
+
 router.get("/*", function(req, res/*, next*/) {
   let limit = parseInt(req.query.limit) || 50,
     id, cursor, index;