From a9549d29a9d8bb5ba85f289fac377990e69f7b40 Mon Sep 17 00:00:00 2001 From: "James P. Ketrenos" Date: Tue, 17 Jan 2023 14:12:48 -0800 Subject: [PATCH] Improved UX Signed-off-by: James P. Ketrenos --- client/package-lock.json | 10 ++++ client/package.json | 1 + client/src/App.css | 18 ++++++- client/src/App.tsx | 74 ++++++++++++++++++++++------ ketrface/cluster.py | 98 ++++++++++++++++++++----------------- ketrface/ketrface/dbscan.py | 46 ++++++++--------- ketrface/ketrface/util.py | 12 +++-- server/routes/photos.js | 25 ++++++++++ 8 files changed, 199 insertions(+), 85 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 1aa3e3e..eff945f 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -17,6 +17,7 @@ "@types/react-dom": "^18.0.10", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-resizable-panels": "^0.0.34", "react-scripts": "5.0.1", "typescript": "^4.9.4", "web-vitals": "^2.1.4" @@ -14226,6 +14227,15 @@ "node": ">=0.10.0" } }, + "node_modules/react-resizable-panels": { + "version": "0.0.34", + "resolved": "https://registry.npmjs.org/react-resizable-panels/-/react-resizable-panels-0.0.34.tgz", + "integrity": "sha512-GGT69jbCiK5Fmw7p9mopb+quX63g+OA235bSHtj8TD3O+wsFNgrg9j5TaRI6auP1J10SBmR0OpJ7tX3K7MFxeg==", + "peerDependencies": { + "react": "^16.14.0 || ^17.0.0 || ^18.0.0", + "react-dom": "^16.14.0 || ^17.0.0 || ^18.0.0" + } + }, "node_modules/react-scripts": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/react-scripts/-/react-scripts-5.0.1.tgz", diff --git a/client/package.json b/client/package.json index 11be15e..6fe5fac 100644 --- a/client/package.json +++ b/client/package.json @@ -13,6 +13,7 @@ "@types/react-dom": "^18.0.10", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-resizable-panels": "^0.0.34", "react-scripts": "5.0.1", "typescript": "^4.9.4", "web-vitals": "^2.1.4" diff --git a/client/src/App.css b/client/src/App.css index 9367a7d..64471b1 100644 --- a/client/src/App.css +++ b/client/src/App.css @@ -19,6 +19,21 @@ div { height: 100%; } +.Resizer { + width: 0.5rem; + border: 1px solid black; +} + +.Explorer { + display: flex; + flex-direction: row; + justify-self: stretch; + align-self: stretch; + height: 100%; + width: auto !important; + flex-grow: 1; +} + .Identities { display: flex; overflow-y: scroll; @@ -71,11 +86,12 @@ div { } .Cluster { + user-select: none; display: flex; flex-direction: column; overflow-y: scroll; - flex-grow: 1; padding: 0.5rem; + height: 100%; } .Cluster .Face.Selected { diff --git a/client/src/App.tsx b/client/src/App.tsx index e117a7d..1f2abf1 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -1,8 +1,36 @@ import React, { useState, useMemo, useEffect } from 'react'; import { useApi } from './useApi'; +import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels"; import './App.css'; +const Photo = ({ photoId }: any) => { + const [image, setImage] = useState(undefined); + + useEffect(() => { + if (photoId === 0) { + return; + } + const fetchImageData = async (image: number) => { + console.log(`Loading photo ${image}`); + const res = await window.fetch(`../api/v1/photos/${image}`); + const data = await res.json(); + setImage(data[0]); + }; + + fetchImageData(photoId); + }, [photoId, setImage]); + + if (image === undefined) { + return <> + } + + return (
); +}; + const Face = ({ faceId, onClick, title }: any) => { const idPath = String(faceId % 100).padStart(2, '0'); return ( @@ -19,10 +47,11 @@ const Face = ({ faceId, onClick, title }: any) => { }; type ClusterProps = { - id: number + id: number, + setImage(image: number): void }; -const Cluster = ({ id }: ClusterProps) => { +const Cluster = ({ id, setImage }: ClusterProps) => { const [identity, setIdentity] = useState(undefined); const { loading, data } = useApi( `../api/v1/identities/${id}` @@ -38,15 +67,22 @@ const Cluster = ({ id }: ClusterProps) => { } }, [data]); - - const relatedFacesJSX = useMemo(() => { - const faceClicked = (e: any, id: any) => { + const faceClicked = async (e: any, id: any) => { if (!identity) { return; } const el = e.currentTarget; const face = identity.relatedFaces.find(item => item.faceId === id); + if (!face) { + return; + } + if (e.shiftKey) { + e.stopPropagation(); + e.preventDefault(); + setImage(face.photoId); + return; + } el.classList.toggle('Selected'); console.log(face); } @@ -59,7 +95,7 @@ const Cluster = ({ id }: ClusterProps) => { onClick={faceClicked} title={face.distance}/> ); - }, [identity]); + }, [identity, setImage]); const lastNameChanged = (e: any) => { setIdentity(Object.assign( @@ -138,6 +174,7 @@ const Cluster = ({ id }: ClusterProps) => { type FaceData = { faceId: number, + photoId: number, lastName: string, firstName: string, middleName: string, @@ -190,6 +227,7 @@ const Identities = ({ identities, setIdentity } : IdentitiesProps) => { const App = () => { const [identities, setIdentities] = useState([]); const [identity, setIdentity] = useState(0); + const [image, setImage] = useState(0); const { loading, data } = useApi( '../api/v1/identities' ); @@ -203,14 +241,22 @@ const App = () => { return (
- { loading &&
Loading...
} - { !loading && identity !== 0 && } - { !loading && identity === 0 &&
- Select identity to edit -
} - { !loading && <> - - } + + + {loading &&
Loading...
} + {!loading && identity !== 0 && } + {!loading && identity === 0 &&
+ Select identity to edit +
} + + + + {image === 0 &&
Select image to view
} + {image !== 0 && } + + + { !loading && }
); diff --git a/ketrface/cluster.py b/ketrface/cluster.py index 8c5f3ea..989e92f 100644 --- a/ketrface/cluster.py +++ b/ketrface/cluster.py @@ -72,6 +72,8 @@ def update_cluster_averages(identities): average = np.add(average, face['descriptors']) average = np.divide(average, len(identity['faces'])) identity['descriptors'] = average + identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply( + average, average))) return identities def load_faces(db_path = db_path): @@ -102,6 +104,8 @@ def load_faces(db_path = db_path): 'focus': focus } face['faces'] = [ face ] + face['sqrtsummul'] = np.sqrt(np.sum(np.multiply( + face['descriptors'], face['descriptors']))) faces.append(face) return faces @@ -155,9 +159,10 @@ def build_straglers(faces): print('Loading faces from database') faces = load_faces() -print(f'{len(faces)} faces loaded') -print('Scanning for clusters') -identities = DBSCAN(faces) # process_faces(faces) +minPts = len(faces) / 100 +eps = 0.2 +print(f'Scanning {len(faces)} faces for clusters (minPts: {minPts}, eps: {eps})') +identities = DBSCAN(faces, minPts = minPts, eps = eps) print(f'{len(identities)} clusters grouped') MAX_CLUSTER_DISTANCE = 0.15 # Used to merge clusters @@ -166,37 +171,41 @@ MAX_EPOCH_DISTANCE = 0.14 # Used to prune outliers # Compute average center for all clusters identities = update_cluster_averages(identities) -removed = -1 -epoch = 1 -# Filter each cluster removing any face that is > cluster_max_distance -# from the average center point of the cluster -while removed != 0: - print(f'Epoch {epoch}...') - epoch += 1 - removed = update_distances(identities, prune = True) - if removed > 0: - print(f'Excluded {removed} faces this epoch') +if False: + removed = -1 + epoch = 1 + # Filter each cluster removing any face that is > cluster_max_distance + # from the average center point of the cluster + while removed != 0: + print(f'Epoch {epoch}...') + epoch += 1 + removed = update_distances(identities, prune = True) + if removed > 0: + print(f'Excluded {removed} faces this epoch') print(f'{len(identities)} identities seeded.') +reduced = identities -# Cluster the clusters... -print('Reducing clusters via DBSCAN') -reduced = DBSCAN(identities, eps = MAX_CLUSTER_DISTANCE, minPts = 2) -if len(reduced) == 0: - reduced = identities -# For each cluster, merge the lists of faces referenced in the cluster's -# "faces" field, which is pointing to clusters (and not actual faces) -for cluster in reduced: - merged = [] - for identity in cluster['faces']: - merged = merged + identity['faces'] - cluster['faces'] = merged +if False: + # Cluster the clusters... + print('Reducing clusters via DBSCAN') + reduced = DBSCAN(identities, eps = MAX_CLUSTER_DISTANCE, minPts = 3) + if len(reduced) == 0: + reduced = identities + # For each cluster, merge the lists of faces referenced in the cluster's + # "faces" field, which is pointing to clusters (and not actual faces) + for cluster in reduced: + merged = [] + for identity in cluster['faces']: + merged = merged + identity['faces'] + cluster['faces'] = merged -# Creating a set containing those faces which have not been bound -# to an identity to recluster them in isolation from the rest of -# the faces -straglers = build_straglers(faces) -reduced = reduced + DBSCAN(straglers) +if False: + # Creating a set containing those faces which have not been bound + # to an identity to recluster them in isolation from the rest of + # the faces + straglers = build_straglers(faces) + reduced = reduced + DBSCAN(straglers) # Build a final cluster with all remaining uncategorized faces if False: @@ -225,20 +234,21 @@ update_distances(reduced) sort_identities(reduced) -# This generates a set of differences between clusters and makes -# a recommendation to merge clusters (outside of DBSCAN) -# -# Worth testing on larger data set -for i, A in enumerate(reduced): - for k, B in enumerate(reduced): - if k < i: - continue - if A == B: - continue - distance = findCosineDistance(A['descriptors'], B['descriptors']) - if distance < MAX_CLUSTER_DISTANCE: - distance = "{:0.4f}".format(distance) - print(f'{A["id"]} to {B["id"]} = {distance}: MERGE') +if False: + # This generates a set of differences between clusters and makes + # a recommendation to merge clusters (outside of DBSCAN) + # + # Worth testing on larger data set + for i, A in enumerate(reduced): + for k, B in enumerate(reduced): + if k < i: + continue + if A == B: + continue + distance = findCosineDistanceBaked(A, B) + if distance < MAX_CLUSTER_DISTANCE: + distance = "{:0.4f}".format(distance) + print(f'{A["id"]} to {B["id"]} = {distance}: MERGE') print('Writing to "auto-clusters.html"') redirect_on(os.path.join(html_path, 'auto-clusters.html')) diff --git a/ketrface/ketrface/dbscan.py b/ketrface/ketrface/dbscan.py index e1d441d..a1f495e 100644 --- a/ketrface/ketrface/dbscan.py +++ b/ketrface/ketrface/dbscan.py @@ -11,10 +11,13 @@ Noise = -2 # Union of two lists of dicts, adding unique elements of B to # end of A def Union(A, B): - for key in B: - if key not in A: - A.append(key) - return A +# 5.012 of 100s sample + return A + [x for x in B if x not in A] +# 5.039 of 100s sample +# for key in B: +# if key not in A: +# A.append(key) +# return A # https://en.wikipedia.org/wiki/DBSCAN # @@ -24,19 +27,24 @@ def DBSCAN(points, eps = MAX_DISTANCE, minPts = MIN_PTS, verbose = True): total = len(points) last = 0 start = time.time() + + # NOTE: A point (P) is only scanned once by RangeQuery for cluster + # inclusion. The internal loop does not need to rescan those points + # as they would have already been in a cluster if minPts is reached. for i, P in enumerate(points): + if verbose == True: new_perc = int(100 * (i+1) / total) now = time.time() if new_perc != perc or now - last > 5: perc = new_perc - print(f'Clustering points {perc}% ({i}/{total} processed) complete with {len(clusters)} identities ({now - start}s).') last = now + print(f'Clustering points {perc}% ({i}/{total} processed) complete with {len(clusters)} identities ({int(now - start)}s).') if P['cluster'] != Undefined: # Previously processed in inner loop continue - N = RangeQuery(points, P, eps) # Find neighbors + N = RangeQuery(points[i:], P, eps)# Find neighbors if len(N) < minPts: # Density check P['cluster'] = Noise # Label as Noise continue @@ -49,8 +57,7 @@ def DBSCAN(points, eps = MAX_DISTANCE, minPts = MIN_PTS, verbose = True): clusters.append(C) P['cluster'] = C # Label initial point - S = N # Neighbors to expand (exclude P) - S.remove(P) + S = N # Neighbors to expand (excludes P) sub_perc = -1 sub_last = 0 @@ -58,13 +65,11 @@ def DBSCAN(points, eps = MAX_DISTANCE, minPts = MIN_PTS, verbose = True): for j, Q in enumerate(S): # Process every seed point if verbose == True: - sub_total = len(S) - sub_new_perc = int(100 * (j+1) / sub_total) sub_now = time.time() - if sub_new_perc != sub_perc or sub_now - sub_last > 5: - sub_perc = sub_new_perc - print(f'... points {sub_perc}% ({j}/{sub_total} processed [{perc}% total]) complete with {len(clusters)} identities ({now - start}s).') + if sub_now - sub_last > 5: + sub_total = len(S) sub_last = sub_now + print(f'... points {j}/{sub_total} processed [{perc}% total]). {len(C["faces"])} forming: {len(clusters)} identities ({int(sub_now - start)}s).') if Q['cluster'] == Noise: # Change Noise to border point Q['cluster'] = C @@ -72,11 +77,11 @@ def DBSCAN(points, eps = MAX_DISTANCE, minPts = MIN_PTS, verbose = True): if Q['cluster'] != Undefined: # Previously processed (border point) continue - + Q['cluster'] = C # Label neighbor C['faces'].append(Q) - N = RangeQuery(points, Q, eps) # Find neighbors + N = RangeQuery(points[i:], Q, eps) # Find neighbors if len(N) >= minPts: # Density check (if Q is a core point) S = Union(S, N) # Add new neighbors to seed set return clusters @@ -86,12 +91,9 @@ def RangeQuery(points, Q, eps): for P in points: # Scan all points in the database if P == Q: continue - if P in neighbors: - continue - distance = findCosineDistance( # Compute distance and check epsilon - Q['descriptors'], - P['descriptors']) - if distance <= eps: - neighbors += [ P ] # Add to result + distance = findCoseinDistanceBaked(# Compute distance + Q, P) + if distance <= eps: # Check epsilon + neighbors.append(P) # Add to result return neighbors diff --git a/ketrface/ketrface/util.py b/ketrface/ketrface/util.py index a639734..c269e28 100644 --- a/ketrface/ketrface/util.py +++ b/ketrface/ketrface/util.py @@ -47,11 +47,15 @@ class NpEncoder(json.JSONEncoder): if isinstance(obj, np.ndarray): return obj.tolist() +def findCoseinDistanceBaked(src, dst): + a = np.matmul(np.transpose(src['descriptors']), dst['descriptors']) + return 1 - (a / (src['sqrtsummul'] * dst['sqrtsummul'])) + def findCosineDistance(source_representation, test_representation): - if type(source_representation) == list: - source_representation = np.array(source_representation) - if type(test_representation) == list: - test_representation = np.array(test_representation) +# if type(source_representation) == list: +# source_representation = np.array(source_representation) +# if type(test_representation) == list: +# test_representation = np.array(test_representation) a = np.matmul(np.transpose(source_representation), test_representation) b = np.sum(np.multiply(source_representation, source_representation)) c = np.sum(np.multiply(test_representation, test_representation)) diff --git a/server/routes/photos.js b/server/routes/photos.js index 366a4d8..0a9598c 100755 --- a/server/routes/photos.js +++ b/server/routes/photos.js @@ -1083,6 +1083,31 @@ console.log("Trying path as: " + path); }); }); +router.get("/:id", async (req, res) => { + const id = parseInt(req.params.id); + try { + const results = await photoDB.sequelize.query( + ` + SELECT photos.*,albums.path AS path, + faces.identityId,faces.top,faces.left,faces.right,faces.bottom + FROM photos + INNER JOIN albums ON albums.id=photos.albumId + INNER JOIN faces ON faces.photoId=photos.id + WHERE photos.id=:id + `, { + replacements: { id } + } + ); + if (results.length === 0) { + return res.status(404); + } + return res.status(200).json(results[0]); + } catch (error) { + console.error(error); + return res.status(404).json({message: `Error connecting to DB for ${id}.`}) + } +}); + router.get("/*", function(req, res/*, next*/) { let limit = parseInt(req.query.limit) || 50, id, cursor, index;