James Ketrenos f1c1b79672 Face selection is working pretty well
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
2023-01-25 22:57:42 -08:00

973 lines
27 KiB
TypeScript

import React, { useState, useMemo, useEffect, useRef, useCallback } from 'react';
import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels";
import {
BrowserRouter as Router,
Route,
Routes,
useParams
} from "react-router-dom";
import { VirtuosoGrid } from 'react-virtuoso'
import moment from 'moment';
import './App.css';
const base = process.env.PUBLIC_URL; /* /identities -- set in .env */
const makeFaceBoxes = (photo: any,
dimensions: any,
onFaceClick: any,
onFaceEnter: any,
onFaceLeave: any,
): any => {
const faces: FaceData[] = photo.faces;
let width: number, height: number, offsetLeft = 0, offsetTop = 0;
/* If photo is wider than viewport, it will be 100% width and < 100% height */
if (photo.width / photo.height > dimensions.width / dimensions.height) {
console.log('Landscape');
width = dimensions.width;
height = dimensions.height * photo.height / photo.width *
dimensions.width / dimensions.height;
offsetLeft = 0;
offsetTop = (dimensions.height - height) * 0.5;
} else {
console.log('Portrait');
width = dimensions.width * photo.width / photo.height *
dimensions.height / dimensions.width;
height = dimensions.height;
offsetLeft = (dimensions.width - width) * 0.5;
offsetTop = 0;
}
return faces.map((face: FaceData) => {
const faceId = face.faceId;
const identityId = face.identityId;
return (
<div className="FaceBox"
key={faceId}
data-identity-id={identityId}
data-face-id={faceId}
style={{
left: offsetLeft + Math.floor(face.left * width) + "px",
top: offsetTop + Math.floor(face.top * height) + "px",
width: Math.floor((face.right - face.left) * width) + "px",
height: Math.floor((face.bottom - face.top) * height) + "px"
}}
onClick={(e) => { onFaceClick(e, face) }}
onMouseEnter={(e) => { onFaceEnter(e, face) }}
onMouseLeave={(e) => { onFaceLeave(e, face) }}
/>
)
});
};
const Photo = ({ photoId, onFaceClick }: any) => {
const [image, setImage] = useState<any>(undefined);
const [faceInfo, setFaceInfo] = useState<string>('');
const ref = useRef(null);
const [dimensions, setDimensions] = React.useState({width: 0, height: 0});
const onFaceEnter = (e: any, face: FaceData) => {
onFaceMouseEnter(e, face);
setFaceInfo(face.identity ? face.identity.displayName : 'Unknown');
}
const onFaceLeave = (e: any, face: FaceData) => {
setFaceInfo('');
onFaceMouseLeave(e, face);
}
const faces = useMemo(() => {
if (image === undefined || dimensions.height === 0) {
return <></>;
}
return makeFaceBoxes(image, dimensions,
onFaceClick, onFaceEnter, onFaceLeave);
}, [image, dimensions, onFaceClick]);
const checkResize = useCallback(() => {
if (!ref.current) {
return;
}
const el: Element = ref.current as Element;
if (dimensions.height !== el.clientHeight
|| dimensions.width !== el.clientWidth) {
setDimensions({
height: el.clientHeight,
width: el.clientWidth
});
}
}, [setDimensions, dimensions]);
useEffect(() => {
let timer = setInterval(() => checkResize(), 250);
return () => { clearInterval(timer); }
}, [checkResize]);
useEffect(() => {
if (photoId === 0) {
return;
}
const fetchImageData = async (image: number) => {
console.log(`Loading photo ${image}`);
const res = await window.fetch(`${base}/api/v1/photos/${image}`);
const photo = await res.json();
setImage(photo);
};
fetchImageData(photoId);
}, [photoId, setImage]);
if (image === undefined) {
return <></>
}
return (<div className="PhotoPanel">
<div className="Image" ref={ref}>
<img
alt={image.filename}
src={`${base}/../${image.path}thumbs/scaled/${image.filename}`.replace(/ /g, '%20')}/>
{ faces }
</div>
<div className="ImageInfo">{
moment(image.taken)
.format('MMMM Do YYYY, h:mm:ss a')
}, {
moment(image.taken)
.fromNow()
}.</div>
<div className="FaceInfo">{ faceInfo ? faceInfo : 'Hover over face for information.'}</div>
</div>
);
};
const onFaceMouseEnter = (e: any, face: FaceData) => {
const faceId = face.faceId;
const els = [...document.querySelectorAll(`[data-face-id="${faceId}"]`)];
const identityId = face.identityId;
els.splice(0, 0,
...document.querySelectorAll(
`.Identities [data-identity-id="${identityId}"]`),
...document.querySelectorAll(
`.Photo [data-identity-id="${identityId}"]`));
els.forEach(el => {
el.classList.add('Active');
});
e.stopPropagation();
e.preventDefault();
};
const onFaceMouseLeave = (e: any, face: FaceData) => {
const faceId = face.faceId;
const els = [...document.querySelectorAll(`[data-face-id="${faceId}"]`)];
const identityId = face.identityId;
els.splice(0, 0,
...document.querySelectorAll(`[data-identity-id="${identityId}"]`));
els.forEach(el => {
el.classList.remove('Active');
});
e.stopPropagation();
e.preventDefault();
};
const Face = ({ face, onFaceClick, title, isSelected }: any) => {
const faceId = face.faceId;
const idPath = String(faceId % 100).padStart(2, '0');
const img = faceId === -1
? <div className='UnknownFace'>?</div>
: <img alt={faceId} src={`${base}/../faces/${idPath}/${faceId}.jpg`}/>;
return (
<div
data-face-id={face.faceId}
data-identity-id={face.identityId}
onClick={(e) => { onFaceClick(e, face) }}
onMouseEnter={(e) => { onFaceMouseEnter(e, face) }}
onMouseLeave={(e) => { onFaceMouseLeave(e, face) }}
className={`Face ${isSelected ? 'Selected' : ''}`}>
<div className='Image'>
{ img }
<div className='Title'>{title}</div>
</div>
</div>
);
};
type ClusterProps = {
identity: IdentityData,
setIdentity(identity: IdentityData): void,
identities: IdentityData[],
setIdentities(identiteis: IdentityData[]): void,
setImage(image: number): void,
selected: number[],
setSelected(selected: number[]): void,
};
const Cluster = ({
identity, setIdentity,
identities, setIdentities,
selected,
setSelected,
setImage,
}: ClusterProps) => {
const [lastName, setLastName] = useState<string>(identity.lastName);
const [firstName, setFirstName] = useState<string>(identity.firstName);
const [middleName, setMiddleName] = useState<string>(identity.middleName);
const [displayName, setDisplayName] = useState<string>(identity.displayName);
const [updated, setUpdated] = useState<boolean>(false);
const lastNameChanged = (e: any) => {
setLastName(e.currentTarget.value);
};
const firstNameChanged = (e: any) => {
setFirstName(e.currentTarget.value);
};
const middleNameChanged = (e: any) => {
setMiddleName(e.currentTarget.value);
};
const displayNameChanged = (e: any) => {
setDisplayName(e.currentTarget.value);
};
/* If the user edits the identity, set the "updated" flag */
useEffect(() => {
setUpdated(lastName !== identity.lastName
|| firstName !== identity.firstName
|| middleName !== identity.middleName
|| displayName !== identity.displayName
);
}, [setUpdated, identity, lastName, firstName, middleName, displayName]);
/* If the identity changes, update all the fields */
useEffect(() => {
setLastName(identity.lastName);
setFirstName(identity.firstName);
setMiddleName(identity.middleName);
setDisplayName(identity.displayName);
}, [identity]);
const faceClicked = useCallback((e: any, face: FaceData) => {
const el = e.currentTarget;
/* Control -- select / deselect single item */
if (e.ctrlKey) {
el.classList.toggle('Selected');
const tmp = [...document.querySelectorAll('.Cluster .Selected')]
.map((face: any) => +face.getAttribute('data-face-id'));
setSelected(tmp);
return;
}
/* Shift -- select groups */
if (e.shiftKey) {
return;
}
/* Default to load image */
e.stopPropagation();
e.preventDefault();
setImage(face.photoId);
}, [setSelected, setImage]);
const deleteIdentity = async () => {
if (!identity || identity.identityId === -1) {
return;
}
try {
const res = await window.fetch(
`${base}/api/v1/identities/${identity.identityId}`, {
method: 'DELETE',
headers: { 'Content-Type': 'application/json' }
});
await res.json();
const index = identities
.findIndex((item: IdentityData) =>
item.identityId === identity.identityId);
if (index !== -1) {
identities.splice(index, 1);
}
setIdentity(EmptyIdentity);
setIdentities([...identities]);
} catch (error) {
console.error(error);
}
};
const updateIdentity = async () => {
if (!identity || identity.identityId === -1) {
return;
}
try {
const values = {
lastName,
firstName,
middleName,
displayName
};
const res = await window.fetch(
`${base}/api/v1/identities/${identity.identityId}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(values)
});
await res.json();
setIdentity({ ...identity, ...values });
} catch (error) {
console.error(error);
}
};
const createIdentity = async () => {
try {
const values = {
lastName,
firstName,
middleName,
displayName
};
const res = await window.fetch(
`${base}/api/v1/identities/`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(values)
});
const created = await res.json();
setIdentity(created);
} catch (error) {
console.error(error);
}
};
if (identity === undefined) {
return (<div className='Cluster'>
Select identity to load.
</div>);
}
return (
<div className='Cluster'>
<div className="Info">
<div style={{ display: "flex", flexDirection: "row", gap: "0.25rem" }}>
<form className="IdentityForm">
<div>Last name:</div>
<input type="text"
value={lastName}
onChange={lastNameChanged}/>
<div>First name:</div>
<input type="text"
value={firstName}
onChange={firstNameChanged} />
<div>Middle name:</div><input type="text"
value={middleName}
onChange={middleNameChanged} />
<div>Display name:</div>
<input type="text"
value={displayName}
onChange={displayNameChanged} />
</form>
<Face
face={{
identityId: identity.identityId,
faceId: identity.faceId
}}
onFaceClick={() => {}}
title={`${displayName} (${identity.facesCount})`} />
</div>
<div className="Actions">
<Button onClick={createIdentity}>Create</Button>
{ identity.identityId !== -1 && <>
{ updated && <Button onClick={updateIdentity}>Update</Button> }
<Button onClick={deleteIdentity}>Delete</Button>
</> }
</div>
</div>
<div>Faces: {identity.relatedFaces.length}</div>
<VirtuosoGrid
data={identity.relatedFaces}
listClassName='Faces'
itemContent={(_index, face) => (
<Face
isSelected={
selected.findIndex(
(x: number) => x === face.faceId) !== -1
}
face={face}
onFaceClick={faceClicked}
title={face.distance} />
)}
/>
</div>
);
};
type FaceData = {
faceId: number,
photoId: number,
identity: IdentityData,
identityId: number,
distance: number,
descriptors: any[],
top: number
right: number,
bottom: number,
left: number
};
type IdentityData = {
lastName: string,
middleName: string,
firstName: string,
descriptors: number[],
identityId: number
displayName: string,
relatedFaces: FaceData[],
facesCount: number,
faceId: number
};
const EmptyIdentity: IdentityData = {
lastName: '',
middleName: '',
firstName: '',
descriptors: [],
identityId: -1,
displayName: '',
relatedFaces: [],
facesCount: 0,
faceId: -1
};
const UnknownFace = {
faceId: -1,
photoId: -1,
identityId: -1,
distance: 0,
descriptors: [],
top: 0,
left: 0,
bottom: 0,
right: 0,
identity: EmptyIdentity
};
interface IdentitiesProps {
identities: IdentityData[],
onFaceClick(e: any, face: FaceData): void
};
const Identities = ({ identities, onFaceClick } : IdentitiesProps) => {
const [jsx, setJsx] = useState<any[]>([]);
useEffect(() => {
setJsx(identities.map((identity) => {
const face = {
faceId: identity.faceId,
identityId: identity.identityId
};
return (
<div
key={identity.identityId}
style={{
display: "flex",
justifyContent: 'center',
alignItems: 'center'
}}>
<Face
face={face}
onFaceClick={onFaceClick}
title={`${identity.displayName} (${identity.facesCount})`}/>
</div>
);
}));
}, [identities, onFaceClick]);
return (
<div className='Identities'>
{ jsx }
</div>
);
};
const Button = ({ onClick, children }: any) => {
return (
<button type="button" onClick={onClick}>
{children}
</button>
);
};
/* returns true if update to identities array occurred */
const updateIdentityReferences = (
identities: IdentityData[],
identity: IdentityData) : boolean => {
if (identity.identityId === -1) {
console.warn('Identity Unknown (-1) attempting to be updated');
return false;
}
const targetIndex = identities.findIndex(
x => x.identityId === identity.identityId);
if (targetIndex === -1) {
identities.push(identity);
return true;
}
const target = identities[targetIndex];
/*
IdentityData fields we check to make sure they are the same:
lastName: string,
middleName: string,
firstName: string,
displayName: string,
facesCount: number,
faceId: number
!identityId: number
!relatedFaces: FaceData[],
!descriptors: number[],
*/
let same = true;
[ 'lastName', 'firstName', 'middleName',
'displayName', 'faceId', 'facesCount', 'faceId' ]
.forEach((field: string) => {
same = same && (target as any)[field] === (identity as any)[field];
});
if (same) {
return false;
}
identities[targetIndex] = {
...identity,
relatedFaces: target.relatedFaces
};
/* relatedFaces is a list of references to identity */
identity.relatedFaces.forEach(face => {
face.identity = identity;
});
return true;
};
const App = () => {
const { identityId, faceId } = useParams();
const [identities, setIdentities] = useState<IdentityData[]>([]);
const [selectedIdentities, setSelectedIdentities] = useState<number[]>([]);
const [identity, setIdentity] = useState<IdentityData>(EmptyIdentity);
const [image, setImage] = useState<number>(0);
const [guess, setGuess] = useState<FaceData|undefined>(undefined);
const [loaded, setLoaded] = useState<boolean>(false);
const [selected, setSelected] = useState<number[]>([]);
/* If 'selected' changes, clear any selected face which is not in the
* selected array. */
useEffect(() => {
[...document.querySelectorAll('.Cluster .Selected')].forEach(el => {
const faceId = el.getAttribute('data-face-id');
if (faceId) {
if (selected.findIndex(item => item === +faceId) === -1) {
el.classList.remove('Selected');
}
}
});
}, [selected]);
const loadIdentity = async (identityId: number) => {
try {
const res = await window.fetch(`${base}/api/v1/identities/${identityId}`);
const data = await res.json();
setIdentity(data[0]);
} catch (error) {
console.error(error);
}
};
/* If the identity changes, update its entry in the identities list
* NOTE: Blocks update to 'Unknown' (-1) fake identity */
useEffect(() => {
if (identity.identityId === -1) {
return;
}
if (!updateIdentityReferences(identities, identity)) {
return;
}
setIdentities(
[...identities]
.sort((A: IdentityData, B: IdentityData) => {
/* Sort the Unknown (-1) identity to the end */
if (A.identityId === -1) {
return +1;
}
if (B.identityId === -1) {
return -1;
}
/* Otherwise sort alphabetically by displayName */
return A.displayName.localeCompare(B.displayName);
})
);
}, [identity, setIdentities, identities]);
/* If the identity changes, scroll it into view in the Identities list */
useEffect(() => {
if (selectedIdentities.length !== 0) {
return;
}
const el = document.querySelector(
`.Identities [data-identity-id="${identity.identityId}"]`);
if (el) {
el.scrollIntoView({
behavior: "smooth",
block: "nearest",
inline: "nearest"
});
}
}, [identity, selectedIdentities]);
useEffect(() => {
if (identityId !== undefined && !isNaN(+identityId)) {
loadIdentity(+identityId);
}
if (faceId !== undefined && !isNaN(+faceId)) {
setImage(+faceId);
}
// eslint-disable-next-line
}, []);
useEffect(() => {
if (identities.length !== 0 || loaded) {
return;
}
const loadIdentities = async () => {
const res = await window.fetch(`${base}/api/v1/identities`, {
headers: { 'Content-Type': 'application/json' },
});
const data = await res.json();
data.forEach((identity: IdentityData) => {
identity.relatedFaces.forEach(face => {
face.identity = identity;
});
});
data.sort((A: IdentityData, B: IdentityData) => {
/* Sort the Unknown (-1) identity to the end */
if (A.identityId === -1) {
return +1;
}
if (B.identityId === -1) {
return -1;
}
/* Otherwise sort alphabetically by displayName */
return A.displayName.localeCompare(B.displayName);
});
setLoaded(true);
setIdentities(data as IdentityData[]);
}
loadIdentities();
}, [identities, setIdentities, setLoaded, loaded]);
const removeFacesFromIdentity = (faceIds: number[]) => {
if (!identity) {
return;
}
const pre = identity.relatedFaces.length;
/* Remove all relatedFaces which are part of the set of removed
* faces */
identity.relatedFaces = identity.relatedFaces.filter(
(face: FaceData) => faceIds.indexOf(face.faceId) === -1);
if (pre !== identity.relatedFaces.length) {
identity.facesCount = identity.relatedFaces.length;
setIdentity({ ...identity })
}
}
const mergeIdentity = async () => {
if (selectedIdentities.length === 0) {
window.alert('You need to select an identity first (CTRL+CLICK)');
return;
}
if (!identity || identity.identityId === -1) {
return;
}
try {
let res = await window.fetch(
`${base}/api/v1/identities/faces/add/${selectedIdentities[0]}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ faces: identity.relatedFaces
.map(face => face.faceId) })
});
const result = await res.json();
res = await window.fetch(
`${base}/api/v1/identities/${identity.identityId}`, {
method: 'DELETE',
headers: { 'Content-Type': 'application/json' }
});
await res.json();
/* Delete the identity from the list of identities */
const deleted = identities
.findIndex((item: IdentityData) =>
item.identityId === identity.identityId);
if (deleted !== -1) {
identities.splice(deleted, 1);
}
/* Update the faces count on the target identity */
const target = identities
.find((item: IdentityData) =>
item.identityId === selectedIdentities[0]);
if (target) {
target.facesCount += result.added.length;
}
setIdentity(EmptyIdentity);
setIdentities([...identities]);
} catch (error) {
console.error(error);
}
};
const removeFaceFromIdentity = async () => {
if (!identity) {
return;
}
try {
const res = await window.fetch(
`${base}/api/v1/identities/faces/remove/${identity.identityId}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ faces: selected })
});
const results = await res.json();
removeFacesFromIdentity(results.removed);
deselectAll();
if (results.faceId !== undefined
&& identity.faceId !== results.faceId) {
setIdentity({...identity, faceId: results.faceId });
}
} catch (error) {
console.error(error);
}
};
const changeSelectedIdentity = async () => {
if (selectedIdentities.length === 0) {
window.alert('You need to select an identity first (CTRL+CLICK)');
return;
}
try {
const res = await window.fetch(
`${base}/api/v1/identities/faces/add/${selectedIdentities[0]}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ faces: selected })
});
const results = await res.json();
removeFacesFromIdentity(results.added);
/* If the identity faceId was removed from the identity, select
* the next relatedFace */
if (results.added.indexOf(identity.faceId) !== -1) {
if (identity.relatedFaces.length === 0) {
identity.faceId = -1;
} else {
identity.faceId = identity.relatedFaces[0].faceId;
}
setIdentity({...identity});
}
/* Update the faces count on the target identity */
const target = identities
.find((item: IdentityData) =>
item.identityId === selectedIdentities[0]);
if (target) {
target.facesCount += results.added.length;
target.faceId = results.faceId;
}
deselectAll();
} catch (error) {
console.error(error);
}
};
const guessIdentity = async () => {
try {
const res = await window.fetch(
`${base}/api/v1/identities/faces/guess/${selected[0]}`, {
method: 'GET',
headers: { 'Content-Type': 'application/json' },
});
const faces = await res.json();
setGuess(faces[0]);
} catch (error) {
console.error(error);
}
};
const updateFasAsNotFace = async () => {
try {
const res = await window.fetch(
`${base}/api/v1/faces`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
action: 'not-a-face',
faces: selected
})
});
await res.json();
removeFacesFromIdentity(selected);
deselectAll();
} catch (error) {
console.error(error);
}
};
const deselectAll = () => {
[...document.querySelectorAll('.Cluster .Selected')].forEach(item => {
item.classList.remove('Selected');
});
setSelected([]);
};
const onFaceClick = (e: any, face: FaceData) => {
const identityId = face.identityId;
const faceId = face.faceId;
console.log(`onFaceClick`, { faceId, identityId});
const faces = [
...document.querySelectorAll(`.Identities [data-identity-id="${identityId}"]`),
...document.querySelectorAll(`.Cluster [data-face-id="${faceId}"]`)];
faces.forEach((el: any) => {
el.scrollIntoView({
behavior: "smooth",
block: "nearest",
inline: "nearest"
});
});
};
const guessOnFaceClick = (e: any, face: FaceData) => {
};
const identitiesOnFaceClick = (e: any, face: FaceData) => {
const identityId = face.identityId;
const el = e.currentTarget;
/* Control -- select / deselect single item */
if (e.ctrlKey) {
let set = !el.classList.contains('Selected');
[...document.querySelectorAll('.Identities .Selected')].forEach(item => {
item.classList.remove('Selected')
});
if (set) {
el.classList.add('Selected');
}
const tmp = [...document.querySelectorAll('.Identities .Selected')]
.map((face: any) => +face.getAttribute('data-identity-id'));
setSelectedIdentities(tmp);
return;
}
/* Shift -- select groups */
if (e.shiftKey) {
return;
}
/* Default to load image */
e.stopPropagation();
e.preventDefault();
if (identity.identityId !== identityId
|| identity.facesCount === 0) {
[...document.querySelectorAll('.Cluster .Faces img')]
.forEach((img: any) => {
img.src = 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7';
});
loadIdentity(identityId);
}
}
return (
<div className="App">
<div className="Worksheet">
<PanelGroup className="Explorer"
autoSaveId="persistence" direction="horizontal">
<Panel defaultSize={50} className="ClusterEditor">
<Cluster {...{
identity,
setIdentity,
identities,
setIdentities,
setImage,
selected,
setSelected,
}} />
<div className="Actions">
{ selected.length === 1 && <>
<Button onClick={guessIdentity}>Guess</Button>
</>}
{ selected.length !== 0 && <>
<Button onClick={removeFaceFromIdentity}>Remove</Button>
<Button onClick={updateFasAsNotFace}>Not a face</Button>
<Button onClick={changeSelectedIdentity}>Change Identity</Button>
<Button onClick={deselectAll}>Deselect All</Button>
</>}
{ selectedIdentities.length !== 0 && <>
<Button onClick={mergeIdentity}>Merge</Button>
</> }
</div>
</Panel>
<PanelResizeHandle className="Resizer"/>
<Panel>
<div className="Viewer">
{image === 0 && <div style={{ margin: '1rem' }}>Select image to view</div>}
{image !== 0 && <Photo onFaceClick={onFaceClick} photoId={image}/> }
{guess !== undefined && guess.identity && <div
className="Guess">
<Face
face={guess.identity.relatedFaces[0]}
onFaceClick={guessOnFaceClick}
title={`${guess.identity.displayName} (${guess.distance})`}/>
</div> }
</div>
</Panel>
<PanelResizeHandle className="Resizer" />
<Panel defaultSize={8.5} minSize={8.5} className="IdentitiesList">
{ !loaded && <div style={{ margin: '1rem' }}>
Loading...
</div> }
{ loaded && <Identities
{...{ onFaceClick: identitiesOnFaceClick, identities }}/>
}
</Panel>
</PanelGroup>
</div>
</div>
);
}
const AppRouter = () => {
return (
<Router>
<Routes>
<Route element={<App />} path={`${base}/:identityId?/:faceId?`} />
</Routes>
</Router>
);
}
export default AppRouter;