Functional for editing clusters; need to add merging
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
parent
17fbb4e21c
commit
1975b174a8
20
README.md
20
README.md
@ -8,6 +8,26 @@ This photo manager performs the following:
|
||||
3. Schedule backend processing of all photos that have not been face
|
||||
scanned with the latest FACE_SCANNER version
|
||||
|
||||
## Populating...
|
||||
|
||||
1. App builds photo DB.
|
||||
This occurs via 'server/app.js' (npm start)
|
||||
2. Faces are identified in all photos.
|
||||
This occurs in 'ketrface/detect.py' (python3 detect.py)
|
||||
1. mtcnn finds faces (threshold > 0.95)
|
||||
2. vgg-face is used to generate detectors
|
||||
3. Faces are clustered into groups.
|
||||
This occurs in 'ketrface/cluster.py' (python3 cluster.py)
|
||||
1. DBSCAN generates first set of clusters
|
||||
2. Centroid calculated for all clusters
|
||||
3. Faces are pruned from clusters if they exceed a threshold
|
||||
4. Clusters are merged based on centroid distances
|
||||
4. Clusters are examined to ensure only one face per image is assigned an identity.
|
||||
This occurs in 'ketrface/split.py' (python3 cluster.py)
|
||||
1. echo "select f1.id,f2.id,f1.photoId,f1.identityId from faces as f1 join faces as f2 on f2.identityId=f1.identityId and f1.photoId=f2.photoId and f1.id!=f2.id;"
|
||||
2. For each face in the same photo, determine which face is closest to the cluster centroid. Move the other to cluster-1.{N} where N increases for each duplicate
|
||||
|
||||
|
||||
# To use the Docker
|
||||
|
||||
Edit the environment file '.env' and set PICTURES to the correct
|
||||
|
||||
@ -36,12 +36,14 @@ div {
|
||||
}
|
||||
|
||||
.Identities {
|
||||
display: flex;
|
||||
padding: 0.25rem;
|
||||
gap: 0.25rem;
|
||||
display: grid;
|
||||
user-select: none;
|
||||
overflow-y: scroll;
|
||||
flex-direction: column;
|
||||
border: 1px solid green;
|
||||
overflow-x: clip;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
gap: 0.25rem;
|
||||
grid-template-columns: repeat(auto-fill, minmax(8.5rem, auto));
|
||||
}
|
||||
|
||||
.Face {
|
||||
@ -49,11 +51,12 @@ div {
|
||||
box-sizing: border-box;
|
||||
flex-direction: column;
|
||||
position: relative;
|
||||
border: 2px solid #444;
|
||||
border: 0.25rem solid #444;
|
||||
}
|
||||
|
||||
.Face:hover {
|
||||
cursor: pointer;
|
||||
border-color: yellow;
|
||||
}
|
||||
|
||||
.ClusterEditor {
|
||||
@ -82,17 +85,19 @@ div {
|
||||
background-position: 50% 50% !important;
|
||||
}
|
||||
|
||||
.Active {
|
||||
filter: brightness(1.25);
|
||||
border-color: orange !important;
|
||||
.IdentityForm {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
}
|
||||
|
||||
.Face:hover {
|
||||
border-color: yellow;
|
||||
.Face.Active,
|
||||
.FaceBox.Active {
|
||||
filter: brightness(1.25);
|
||||
border-color: orange;
|
||||
}
|
||||
|
||||
.Face.Selected {
|
||||
border-color: blue;
|
||||
border-color: blue !important;
|
||||
}
|
||||
|
||||
.Face .Title {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
|
||||
import React, { useState, useMemo, useEffect, useRef } from 'react';
|
||||
import React, { useState, useMemo, useEffect, useRef, useCallback } from 'react';
|
||||
import { useApi } from './useApi';
|
||||
import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels";
|
||||
import {
|
||||
@ -66,10 +66,11 @@ const Photo = ({ photoId, onFaceClick }: any) => {
|
||||
return makeFaceBoxes(image, dimensions, onFaceClick);
|
||||
}, [image, dimensions, onFaceClick]);
|
||||
|
||||
useEffect(() => {
|
||||
const checkResize = useCallback(() => {
|
||||
if (!ref.current) {
|
||||
return;
|
||||
}
|
||||
|
||||
const el: Element = ref.current as Element;
|
||||
if (dimensions.height !== el.clientHeight
|
||||
|| dimensions.width !== el.clientWidth) {
|
||||
@ -78,7 +79,13 @@ const Photo = ({ photoId, onFaceClick }: any) => {
|
||||
width: el.clientWidth
|
||||
})
|
||||
}
|
||||
}/*, [dimensions.height, dimensions.width]*/);
|
||||
}, [setDimensions, dimensions]);
|
||||
|
||||
|
||||
useEffect(() => {
|
||||
let timer = setInterval(() => checkResize(), 250);
|
||||
return () => { clearInterval(timer); }
|
||||
}, [checkResize]);
|
||||
|
||||
useEffect(() => {
|
||||
if (photoId === 0) {
|
||||
@ -98,11 +105,16 @@ const Photo = ({ photoId, onFaceClick }: any) => {
|
||||
return <></>
|
||||
}
|
||||
|
||||
return (<div className="Image"
|
||||
ref={ref}
|
||||
style={{
|
||||
background: `url("${base}/../${image.path}thumbs/scaled/${image.filename}")`.replace(/ /g, '%20')
|
||||
}}>{ faces }</div>
|
||||
return (<div className="Image" ref={ref}>
|
||||
<img
|
||||
src={`${base}/../${image.path}thumbs/scaled/${image.filename}`.replace(/ /g, '%20')}
|
||||
style={{
|
||||
objectFit: 'contain',
|
||||
width: '100%',
|
||||
height: '100%'
|
||||
}} />
|
||||
{ faces }
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@ -113,7 +125,10 @@ const onFaceMouseEnter = (e: any, face: FaceData) => {
|
||||
if (face.identity) {
|
||||
const identityId = face.identity.identityId;
|
||||
els.splice(0, 0,
|
||||
...document.querySelectorAll(`[data-identity-id="${identityId}"]`));
|
||||
...document.querySelectorAll(
|
||||
`.Identities [data-identity-id="${identityId}"]`),
|
||||
...document.querySelectorAll(
|
||||
`.Photo [data-identity-id="${identityId}"]`));
|
||||
}
|
||||
|
||||
els.forEach(el => {
|
||||
@ -148,10 +163,13 @@ const Face = ({ face, onFaceClick, title, ...rest }: any) => {
|
||||
onMouseEnter={(e) => { onFaceMouseEnter(e, face) }}
|
||||
onMouseLeave={(e) => { onFaceMouseLeave(e, face) }}
|
||||
className='Face'>
|
||||
<div className='Image'
|
||||
style={{
|
||||
background: `url("${base}/../faces/${idPath}/${faceId}.jpg")`,
|
||||
}}>
|
||||
<div className='Image'>
|
||||
<img src={`${base}/../faces/${idPath}/${faceId}.jpg`}
|
||||
style={{
|
||||
objectFit: 'contain',
|
||||
width: '100%',
|
||||
height: '100%'
|
||||
}}/>
|
||||
<div className='Title'>{title}</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -166,6 +184,8 @@ type ClusterProps = {
|
||||
};
|
||||
|
||||
const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps) => {
|
||||
console.log(identity);
|
||||
|
||||
const relatedFacesJSX = useMemo(() => {
|
||||
const faceClicked = async (e: any, face: FaceData) => {
|
||||
if (!identity) {
|
||||
@ -175,9 +195,13 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps)
|
||||
|
||||
/* Control -- select / deselect single item */
|
||||
if (e.ctrlKey) {
|
||||
const cluster = document.querySelector('.Cluster');
|
||||
el.classList.toggle('Selected');
|
||||
const selected = [...el.parentElement
|
||||
.querySelectorAll('.Selected')]
|
||||
if (!cluster) {
|
||||
return;
|
||||
}
|
||||
|
||||
const selected = [...cluster.querySelectorAll('.Selected')]
|
||||
.map((face: any) => face.getAttribute('data-face-id'));
|
||||
setSelected(selected);
|
||||
return;
|
||||
@ -196,51 +220,57 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps)
|
||||
if (identity === undefined) {
|
||||
return <></>;
|
||||
}
|
||||
return identity.relatedFaces.map(face =>
|
||||
<div
|
||||
key={face.faceId}
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center"}}>
|
||||
<Face
|
||||
face={face}
|
||||
onFaceClick={faceClicked}
|
||||
title={face.distance}/>
|
||||
</div>
|
||||
);
|
||||
return identity.relatedFaces.map((face: FaceData) => {
|
||||
return (
|
||||
<div
|
||||
key={face.faceId}
|
||||
style={{
|
||||
display: "flex",
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center'}}>
|
||||
<Face
|
||||
face={face}
|
||||
onFaceClick={faceClicked}
|
||||
title={face.distance}/>
|
||||
</div>
|
||||
);
|
||||
});
|
||||
}, [identity, setImage, setSelected]);
|
||||
|
||||
const lastNameChanged = (e: any) => {
|
||||
setIdentity(Object.assign(
|
||||
{},
|
||||
identity, {
|
||||
lastName: e.currentTarget.value
|
||||
}
|
||||
));
|
||||
setIdentity({...identity, lastName: e.currentTarget.value });
|
||||
};
|
||||
const firstNameChanged = (e: any) => {
|
||||
setIdentity(Object.assign(
|
||||
{},
|
||||
identity, {
|
||||
firstName: e.currentTarget.value
|
||||
}
|
||||
));
|
||||
setIdentity({...identity, firstName: e.currentTarget.value });
|
||||
};
|
||||
const middleNameChanged = (e: any) => {
|
||||
setIdentity(Object.assign(
|
||||
{},
|
||||
identity, {
|
||||
middleName: e.currentTarget.value
|
||||
}
|
||||
));
|
||||
setIdentity({...identity, middleName: e.currentTarget.value });
|
||||
};
|
||||
const displayNameChanged = (e: any) => {
|
||||
setIdentity(Object.assign(
|
||||
{},
|
||||
identity, {
|
||||
displayName: e.currentTarget.value
|
||||
setIdentity({...identity, displayName: e.currentTarget.value });
|
||||
};
|
||||
|
||||
const updateIdentity = async () => {
|
||||
try {
|
||||
const validFields = [
|
||||
'id', 'displayName', 'firstName', 'lastName', 'middleName'];
|
||||
const filtered: any = Object.assign({}, identity);
|
||||
for (let key in filtered) {
|
||||
if (validFields.indexOf(key) == -1) {
|
||||
delete filtered[key]
|
||||
}
|
||||
}
|
||||
const res = await window.fetch(
|
||||
`${base}/api/v1/identities/${identity.identityId}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(filtered)
|
||||
});
|
||||
const data = await res.json();
|
||||
setIdentity({ ...identity });
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
}
|
||||
));
|
||||
};
|
||||
|
||||
if (identity === undefined) {
|
||||
@ -251,27 +281,27 @@ const Cluster = ({ identity, setIdentity, setImage, setSelected }: ClusterProps)
|
||||
|
||||
return (
|
||||
<div className='Cluster'>
|
||||
<div className="Info">
|
||||
<form>
|
||||
<div><label>Last name:<input type="text"
|
||||
value={identity.lastName}
|
||||
onChange={lastNameChanged}/>
|
||||
</label></div>
|
||||
<div><label>First name:<input type="text"
|
||||
value={identity.firstName}
|
||||
onChange={firstNameChanged} />
|
||||
</label></div>
|
||||
<div><label>Middle name:<input type="text"
|
||||
value={identity.middleName}
|
||||
onChange={middleNameChanged} />
|
||||
</label></div>
|
||||
<div><label>Display name:<input type="text"
|
||||
value={identity.displayName}
|
||||
onChange={displayNameChanged} />
|
||||
</label></div>
|
||||
<div>Faces: {identity.relatedFaces.length}</div>
|
||||
</form>
|
||||
<div className="Info">
|
||||
<form className="IdentityForm">
|
||||
<div>Last name:</div>
|
||||
<input type="text"
|
||||
value={identity.lastName}
|
||||
onChange={lastNameChanged}/>
|
||||
<div>First name:</div>
|
||||
<input type="text"
|
||||
value={identity.firstName}
|
||||
onChange={firstNameChanged} />
|
||||
<div>Middle name:</div><input type="text"
|
||||
value={identity.middleName}
|
||||
onChange={middleNameChanged} />
|
||||
<div>Display name:</div>
|
||||
<input type="text"
|
||||
value={identity.displayName}
|
||||
onChange={displayNameChanged} />
|
||||
</form>
|
||||
<Button onClick={updateIdentity}>Update</Button>
|
||||
</div>
|
||||
<div>Faces: {identity.relatedFaces.length}</div>
|
||||
<div className="Faces">
|
||||
{ relatedFacesJSX }
|
||||
</div>
|
||||
@ -316,10 +346,18 @@ const Identities = ({ identities, onFaceClick } : IdentitiesProps) => {
|
||||
return identities.map((identity) => {
|
||||
const face = identity.relatedFaces[0];
|
||||
return (
|
||||
<Face key={face.faceId}
|
||||
face={face}
|
||||
onFaceClick={onFaceClick}
|
||||
title={identity.displayName}/>
|
||||
<div
|
||||
key={face.faceId}
|
||||
style={{
|
||||
display: "flex",
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center'
|
||||
}}>
|
||||
<Face
|
||||
face={face}
|
||||
onFaceClick={onFaceClick}
|
||||
title={identity.displayName}/>
|
||||
</div>
|
||||
);
|
||||
});
|
||||
}, [ identities, onFaceClick ]);
|
||||
@ -342,7 +380,7 @@ const Button = ({ onClick, children }: any) => {
|
||||
const App = () => {
|
||||
const [identities, setIdentities] = useState<IdentityData[]>([]);
|
||||
const { identityId, faceId } = useParams();
|
||||
const [identity, setIdentity] = useState<any>(undefined);
|
||||
const [identity, setIdentity] = useState<IdentityData | undefined>(undefined);
|
||||
const [image, setImage] = useState<number>(0);
|
||||
const { loading, data } = useApi(
|
||||
`${base}/api/v1/identities`
|
||||
@ -359,6 +397,31 @@ const App = () => {
|
||||
}
|
||||
};
|
||||
|
||||
/* If the identity changes, update its entry in the identities list */
|
||||
useEffect(() => {
|
||||
if (!identity || identities.length === 0) {
|
||||
return;
|
||||
}
|
||||
for (let key in identities) {
|
||||
if (identities[key].identityId === identity.identityId) {
|
||||
if (identities[key] !== identity) {
|
||||
/* Inherit all fields from prior identity, while
|
||||
* also keeping the same reference pointer */
|
||||
identities[key] = {
|
||||
...identity,
|
||||
relatedFaces: identities[key].relatedFaces
|
||||
};
|
||||
/* relatedFaces is a list of references to identity */
|
||||
identity.relatedFaces.forEach(face => {
|
||||
face.identity = identity;
|
||||
});
|
||||
setIdentities([...identities]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}, [identity, setIdentities, identities]);
|
||||
|
||||
useEffect(() => {
|
||||
if (identityId !== undefined && !isNaN(+identityId)) {
|
||||
loadIdentity(+identityId);
|
||||
@ -367,6 +430,7 @@ const App = () => {
|
||||
if (faceId !== undefined && !isNaN(+faceId)) {
|
||||
setImage(+faceId);
|
||||
}
|
||||
// eslint-disable-next-line
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
@ -380,7 +444,24 @@ const App = () => {
|
||||
}
|
||||
}, [data]);
|
||||
|
||||
const removeSelected = async () => {
|
||||
const removeFacesFromIdentities = (faceIds: number[]) => {
|
||||
if (!identity) {
|
||||
return;
|
||||
}
|
||||
const pre = identity.relatedFaces.length;
|
||||
/* Remove all relatedFaces which are part of the set of removed
|
||||
* faces */
|
||||
identity.relatedFaces = identity.relatedFaces.filter(
|
||||
(face: FaceData) => faceIds.indexOf(face.faceId) === -1);
|
||||
if (pre !== identity.relatedFaces.length) {
|
||||
setIdentity({ ...identity })
|
||||
}
|
||||
}
|
||||
|
||||
const markSelectedIncorrectIdentity = async () => {
|
||||
if (!identity) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const res = await window.fetch(
|
||||
`${base}/api/v1/identities/faces/remove/${identity.identityId}`, {
|
||||
@ -390,14 +471,25 @@ const App = () => {
|
||||
});
|
||||
const data = await res.json();
|
||||
|
||||
const pre = identity.relatedFaces.length;
|
||||
/* Remove all relatedFaces which are part of the set of removed
|
||||
* faces */
|
||||
identity.relatedFaces = identity.relatedFaces.filter(
|
||||
(face: FaceData) => data.faces.indexOf(face.faceId) === -1);
|
||||
if (pre !== identity.relatedFaces.length) {
|
||||
setIdentity({...identity})
|
||||
}
|
||||
removeFacesFromIdentities(data.faces);
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
}
|
||||
};
|
||||
|
||||
const markSelectedNotFace = async () => {
|
||||
try {
|
||||
const res = await window.fetch(
|
||||
`${base}/api/v1/faces`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
action: 'not-a-face',
|
||||
faces: selected
|
||||
})
|
||||
});
|
||||
const data = await res.json();
|
||||
removeFacesFromIdentities(data);
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
}
|
||||
@ -409,17 +501,14 @@ const App = () => {
|
||||
return;
|
||||
}
|
||||
const identityId = face.identity.identityId;
|
||||
console.log(face.identity);
|
||||
const identitiesEl = document.querySelector('.Identities');
|
||||
if (!identitiesEl) {
|
||||
return;
|
||||
}
|
||||
const faceIdentity = identitiesEl.querySelector(
|
||||
`[data-identity-id="${identityId}"]`);
|
||||
if (!faceIdentity) {
|
||||
return;
|
||||
}
|
||||
faceIdentity.scrollIntoView()
|
||||
const faceId = face.faceId;
|
||||
console.log(`onFaceClick`, { faceId, identityId});
|
||||
const faces = [
|
||||
...document.querySelectorAll(`.Identities [data-identity-id="${identityId}"]`),
|
||||
...document.querySelectorAll(`.Cluster [data-face-id="${faceId}"]`)];
|
||||
faces.forEach((el: any) => {
|
||||
el.scrollIntoView();
|
||||
});
|
||||
};
|
||||
|
||||
const identitiesOnFaceClick = (e: any, face: FaceData) => {
|
||||
@ -434,18 +523,21 @@ const App = () => {
|
||||
autoSaveId="persistence" direction="horizontal">
|
||||
<Panel defaultSize={50} className="ClusterEditor">
|
||||
{loading && <div style={{ margin: '1rem' }}>Loading...</div>}
|
||||
{!loading && identity !== 0 &&
|
||||
{!loading && identity !== undefined &&
|
||||
<Cluster {...{
|
||||
identity,
|
||||
setIdentity,
|
||||
setImage,
|
||||
setSelected
|
||||
}} />}
|
||||
{!loading && identity === 0 && <div className="Cluster">
|
||||
{!loading && identity === undefined && <div className="Cluster">
|
||||
Select identity to edit
|
||||
</div>}
|
||||
<div className="Actions">
|
||||
{ selected.length !== 0 && <Button onClick={removeSelected}>Remove</Button> }
|
||||
{ selected.length !== 0 && <>
|
||||
<Button onClick={markSelectedIncorrectIdentity}>Remove</Button>
|
||||
<Button onClick={markSelectedNotFace}>Not a face</Button>
|
||||
</>}
|
||||
</div>
|
||||
</Panel>
|
||||
<PanelResizeHandle className="Resizer"/>
|
||||
@ -453,9 +545,12 @@ const App = () => {
|
||||
{image === 0 && <div style={{ margin: '1rem' }}>Select image to view</div>}
|
||||
{image !== 0 && <Photo onFaceClick={onFaceClick} photoId={image}/> }
|
||||
</Panel>
|
||||
</PanelGroup>
|
||||
{ !loading && <Identities
|
||||
{... { onFaceClick: identitiesOnFaceClick, identities }}/> }
|
||||
<PanelResizeHandle className="Resizer" />
|
||||
<Panel defaultSize={8.5} minSize={8.5} className="IdentitiesList">
|
||||
{ !loading && <Identities
|
||||
{... { onFaceClick: identitiesOnFaceClick, identities }}/> }
|
||||
</Panel>
|
||||
</PanelGroup>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@ -27,19 +27,6 @@ if html_base == "/":
|
||||
MAX_CLUSTER_DISTANCE = 0.14 # Used to merge clusters
|
||||
MAX_DISTANCE_FROM_CENTROID = 0.14 # Used to prune outliers
|
||||
|
||||
# TODO
|
||||
# Switch to using DBSCAN
|
||||
#
|
||||
# Thoughts for determining number of clusters to try and target...
|
||||
#
|
||||
# Augment DBSCAN to rule out identity matching for the same face
|
||||
# appearing more than once in a photo
|
||||
#
|
||||
# NOTE: This means twins or reflections won't both identify in the
|
||||
# same photo -- those faces would then identify as a second face pairing
|
||||
# which could merge with a cluster, but can not be used to match
|
||||
|
||||
|
||||
def gen_html(identities):
|
||||
for identity in identities:
|
||||
|
||||
@ -79,41 +66,6 @@ def update_cluster_averages(identities):
|
||||
average, average)))
|
||||
return identities
|
||||
|
||||
def load_faces(db_path = db_path):
|
||||
print(f'Connecting to database: {db_path}')
|
||||
conn = create_connection(db_path)
|
||||
faces = []
|
||||
with conn:
|
||||
print('Querying faces')
|
||||
cur = conn.cursor()
|
||||
res = cur.execute('''
|
||||
SELECT faces.id,facedescriptors.descriptors,faces.faceConfidence,faces.photoId,faces.focus
|
||||
FROM faces
|
||||
INNER JOIN photos ON (photos.duplicate == 0 OR photos.duplicate IS NULL)
|
||||
JOIN facedescriptors ON (faces.descriptorId=facedescriptors.id)
|
||||
WHERE faces.identityId IS null AND faces.faceConfidence>0.99
|
||||
AND faces.photoId=photos.id
|
||||
''')
|
||||
for row in res.fetchall():
|
||||
id, descriptors, confidence, photoId, focus = row
|
||||
if focus is None:
|
||||
focus = 100 # Assume full focus if focus not set
|
||||
face = {
|
||||
'id': id,
|
||||
'type': 'face',
|
||||
'confidence': confidence,
|
||||
'distance': 0,
|
||||
'photoId': photoId,
|
||||
'descriptors': np.frombuffer(descriptors),
|
||||
'cluster': Undefined,
|
||||
'focus': focus
|
||||
}
|
||||
face['faces'] = [ face ]
|
||||
face['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
|
||||
face['descriptors'], face['descriptors'])))
|
||||
faces.append(face)
|
||||
return faces
|
||||
|
||||
def update_distances(identities,
|
||||
prune = False,
|
||||
maxDistance = MAX_DISTANCE_FROM_CENTROID):
|
||||
@ -169,7 +121,7 @@ def build_straglers(faces):
|
||||
return noise + undefined
|
||||
|
||||
print('Loading faces from database')
|
||||
faces = load_faces()
|
||||
faces = load_faces(db_path = db_path)
|
||||
|
||||
minPts = max(len(faces) / 500, 5)
|
||||
eps = 0.185
|
||||
@ -267,41 +219,6 @@ redirect_on(os.path.join(html_path, 'auto-clusters.html'))
|
||||
gen_html(reduced)
|
||||
redirect_off()
|
||||
|
||||
def create_identity(conn, identity):
|
||||
"""
|
||||
Create a new identity in the identities table
|
||||
:param conn:
|
||||
:param identity:
|
||||
:return: identity id
|
||||
"""
|
||||
sql = '''
|
||||
INSERT INTO identities(descriptors,displayName)
|
||||
VALUES(?,?)
|
||||
'''
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, (
|
||||
np.array(identity['descriptors']),
|
||||
f'cluster-{identity["id"]}'
|
||||
))
|
||||
conn.commit()
|
||||
return cur.lastrowid
|
||||
|
||||
def update_face_identity(conn, faceId, identityId = None):
|
||||
"""
|
||||
Update the identity associated with this face
|
||||
:param conn:
|
||||
:param faceId:
|
||||
:param identityId:
|
||||
:return: None
|
||||
"""
|
||||
sql = '''
|
||||
UPDATE faces SET identityId=? WHERE id=?
|
||||
'''
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, (identityId, faceId))
|
||||
conn.commit()
|
||||
return None
|
||||
|
||||
print(f'Connecting to database: {db_path}')
|
||||
conn = create_connection(db_path)
|
||||
with conn:
|
||||
|
||||
@ -76,3 +76,83 @@ def update_face_count(conn, photoId, faces):
|
||||
cur.execute(sql, (faces, photoId))
|
||||
conn.commit()
|
||||
return None
|
||||
|
||||
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import piexif
|
||||
import sqlite3
|
||||
from sqlite3 import Error
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
|
||||
def load_faces(db_path ):
|
||||
print(f'Connecting to database: {db_path}')
|
||||
conn = create_connection(db_path)
|
||||
faces = []
|
||||
with conn:
|
||||
print('Querying faces')
|
||||
cur = conn.cursor()
|
||||
res = cur.execute('''
|
||||
SELECT faces.id,facedescriptors.descriptors,faces.faceConfidence,faces.photoId,faces.focus
|
||||
FROM faces
|
||||
INNER JOIN photos ON (photos.duplicate == 0 OR photos.duplicate IS NULL)
|
||||
JOIN facedescriptors ON (faces.descriptorId=facedescriptors.id)
|
||||
WHERE faces.identityId IS null AND faces.faceConfidence>0.99
|
||||
AND faces.photoId=photos.id
|
||||
''')
|
||||
for row in res.fetchall():
|
||||
id, descriptors, confidence, photoId, focus = row
|
||||
if focus is None:
|
||||
focus = 100 # Assume full focus if focus not set
|
||||
face = {
|
||||
'id': id,
|
||||
'type': 'face',
|
||||
'confidence': confidence,
|
||||
'distance': 0,
|
||||
'photoId': photoId,
|
||||
'descriptors': np.frombuffer(descriptors),
|
||||
'cluster': 0, # Undefined from dbscan.py
|
||||
'focus': focus
|
||||
}
|
||||
face['faces'] = [ face ]
|
||||
face['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
|
||||
face['descriptors'], face['descriptors'])))
|
||||
faces.append(face)
|
||||
return faces
|
||||
|
||||
def create_identity(conn, identity):
|
||||
"""
|
||||
Create a new identity in the identities table
|
||||
:param conn:
|
||||
:param identity:
|
||||
:return: identity id
|
||||
"""
|
||||
sql = '''
|
||||
INSERT INTO identities(descriptors,displayName)
|
||||
VALUES(?,?)
|
||||
'''
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, (
|
||||
np.array(identity['descriptors']),
|
||||
f'cluster-{identity["id"]}'
|
||||
))
|
||||
conn.commit()
|
||||
return cur.lastrowid
|
||||
|
||||
def update_face_identity(conn, faceId, identityId = None):
|
||||
"""
|
||||
Update the identity associated with this face
|
||||
:param conn:
|
||||
:param faceId:
|
||||
:param identityId:
|
||||
:return: None
|
||||
"""
|
||||
sql = '''
|
||||
UPDATE faces SET identityId=? WHERE id=?
|
||||
'''
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, (identityId, faceId))
|
||||
conn.commit()
|
||||
return None
|
||||
|
||||
255
ketrface/split.py
Normal file
255
ketrface/split.py
Normal file
@ -0,0 +1,255 @@
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import piexif
|
||||
import sqlite3
|
||||
from sqlite3 import Error
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
|
||||
import functools
|
||||
|
||||
from ketrface.util import *
|
||||
from ketrface.dbscan import *
|
||||
from ketrface.db import *
|
||||
from ketrface.config import *
|
||||
|
||||
MAX_DISTANCE_FROM_CENTROID = 0.14 # Used to prune outliers
|
||||
|
||||
config = read_config()
|
||||
|
||||
html_path = merge_config_path(config['path'], 'frontend')
|
||||
pictures_path = merge_config_path(config['path'], config['picturesPath'])
|
||||
faces_path = merge_config_path(config['path'], config['facesPath'])
|
||||
db_path = merge_config_path(config['path'], config["db"]["photos"]["host"])
|
||||
html_base = config['basePath']
|
||||
if html_base == "/":
|
||||
html_base = "."
|
||||
|
||||
def update_cluster_averages(identities):
|
||||
for identity in identities:
|
||||
average = []
|
||||
for face in identity['faces']:
|
||||
if len(average) == 0:
|
||||
average = face['descriptors']
|
||||
else:
|
||||
average = np.add(average, face['descriptors'])
|
||||
average = np.divide(average, len(identity['faces']))
|
||||
identity['descriptors'] = average
|
||||
identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
|
||||
average, average)))
|
||||
return identities
|
||||
|
||||
|
||||
def sort_identities(identities):
|
||||
identities.sort(reverse = True, key = lambda x: len(x['faces']))
|
||||
for identity in identities:
|
||||
identity['faces'].sort(reverse = False, key = lambda x: x['distance'])
|
||||
|
||||
def cluster_sort(A, B):
|
||||
diff = A['cluster'] - B['cluster']
|
||||
if diff > 0:
|
||||
return 1
|
||||
elif diff < 0:
|
||||
return -1
|
||||
diff = A['confidence'] - B['confidence']
|
||||
if diff > 0:
|
||||
return 1
|
||||
elif diff < 0:
|
||||
return -1
|
||||
return 0
|
||||
|
||||
def load_identities(db_path):
|
||||
conn = create_connection(db_path)
|
||||
identities = []
|
||||
with conn:
|
||||
cur = conn.cursor()
|
||||
res = cur.execute('''
|
||||
SELECT
|
||||
identities.id as identityId,
|
||||
identities.displayName as displayName,
|
||||
identities.descriptors as descriptors,
|
||||
COUNT(faces.id) AS faceCount
|
||||
FROM identities
|
||||
JOIN faces ON faces.identityId=identities.id
|
||||
GROUP BY identities.id
|
||||
''')
|
||||
for row in res.fetchall():
|
||||
identityId, displayName, descriptors, faceCount = row
|
||||
identity = {
|
||||
'identityId': identityId,
|
||||
'displayName': displayName,
|
||||
'descriptors': np.frombuffer(descriptors),
|
||||
'faceCount': faceCount,
|
||||
'updated': False
|
||||
}
|
||||
# Pre-bake computations for cosine distance
|
||||
identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
|
||||
identity['descriptors'], identity['descriptors'])))
|
||||
identities.append(identity)
|
||||
return identities
|
||||
|
||||
def find_identity(identities, identityId):
|
||||
for element in identities:
|
||||
if element['identityId'] == identityId:
|
||||
return element
|
||||
raise Exception(f'Identity {identityId} missing')
|
||||
|
||||
def load_doppelganger_photos(db_path):
|
||||
conn = create_connection(db_path)
|
||||
photos = {}
|
||||
with conn:
|
||||
cur = conn.cursor()
|
||||
res = cur.execute('''
|
||||
SELECT
|
||||
f1.identityId AS identityId,
|
||||
photos.id AS photoId,
|
||||
f1.id AS f1_id,
|
||||
f2.id AS f2_id,
|
||||
f1_descriptors.descriptors AS f1_descriptors,
|
||||
f2_descriptors.descriptors AS f2_descriptors
|
||||
FROM faces AS f1
|
||||
INNER JOIN faces AS f2 ON (
|
||||
f2.identityId=f1.identityId AND f1.photoId=f2.photoId and f1_id!=f2_id)
|
||||
INNER JOIN photos ON (
|
||||
photos.duplicate == 0 OR photos.duplicate IS NULL)
|
||||
INNER JOIN facedescriptors AS f1_descriptors ON (
|
||||
f1.descriptorId=f1_descriptors.id)
|
||||
INNER JOIN facedescriptors AS f2_descriptors ON (
|
||||
f2.descriptorId=f2_descriptors.id)
|
||||
WHERE f1.identityId IS NOT NULL AND f1.photoId=photos.id
|
||||
ORDER BY photos.id,f1.identityId
|
||||
''')
|
||||
for row in res.fetchall():
|
||||
identityId, photoId, f1_id, f2_id,f1_descriptors, f2_descriptors = row
|
||||
face1 = {
|
||||
'id': f1_id,
|
||||
'type': 'face',
|
||||
'distance': 0,
|
||||
'descriptors': np.frombuffer(f1_descriptors),
|
||||
'cluster': identityId, # Undefined from dbscan.py
|
||||
}
|
||||
face1['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
|
||||
face1['descriptors'], face1['descriptors'])))
|
||||
|
||||
face2 = {
|
||||
'id': f2_id,
|
||||
'type': 'face',
|
||||
'distance': 0,
|
||||
'descriptors': np.frombuffer(f2_descriptors),
|
||||
'cluster': identityId, # Undefined from dbscan.py
|
||||
}
|
||||
face2['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
|
||||
face2['descriptors'], face2['descriptors'])))
|
||||
|
||||
if photoId not in photos:
|
||||
photos[photoId] = {
|
||||
'photoId': photoId,
|
||||
'dopplegangers': {}
|
||||
}
|
||||
if identityId not in photos[photoId]['dopplegangers']:
|
||||
photos[photoId]['dopplegangers'][identityId] = {
|
||||
'identity': None,
|
||||
'faces': []
|
||||
}
|
||||
|
||||
faceList = photos[photoId]['dopplegangers'][identityId]['faces']
|
||||
for face in [ face1, face2 ]:
|
||||
match = False
|
||||
for key in faceList:
|
||||
if face['id'] == key['id']:
|
||||
match = True
|
||||
break
|
||||
if not match:
|
||||
faceList.append(face)
|
||||
|
||||
return photos
|
||||
|
||||
def remove_face_from_identity(identity, face):
|
||||
identity['updated'] = True
|
||||
average = identity['descriptors']
|
||||
average = np.dot(average, identity['faceCount'])
|
||||
average = np.subtract(average, face['descriptors'])
|
||||
identity['faceCount'] -= 1
|
||||
average = np.divide(average, identity['faceCount'])
|
||||
identity['descriptors'] = average
|
||||
identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
|
||||
average, average)))
|
||||
|
||||
face['identity'] = None
|
||||
face['identityId'] = -1
|
||||
face['distance'] = 0
|
||||
|
||||
print('Loading identities from database...')
|
||||
identities = load_identities(db_path)
|
||||
print(f'{len(identities)} identities loaded.')
|
||||
print('Loading dopplegangers from database...')
|
||||
photos = load_doppelganger_photos(db_path)
|
||||
print(f'{len(photos)} photos with dopplegangers loaded.')
|
||||
print('Binding dopplegangers to identities...')
|
||||
|
||||
face_updates = []
|
||||
identity_updates = []
|
||||
for photoId in photos:
|
||||
photo = photos[photoId]
|
||||
print(f'Processing photo {photoId}...')
|
||||
for identityId in photo['dopplegangers']:
|
||||
if photo['dopplegangers'][identityId]['identity'] == None:
|
||||
photo['dopplegangers'][identityId]['identity'] = find_identity(identities, identityId)
|
||||
faces = photo['dopplegangers'][identityId]['faces']
|
||||
identity = photo['dopplegangers'][identityId]['identity']
|
||||
|
||||
for face in faces:
|
||||
face['identity'] = identity
|
||||
face['distance'] = findCosineDistanceBaked(face, identity)
|
||||
faces.sort(reverse = False, key = lambda x: x['distance'])
|
||||
|
||||
# First face closest to identity -- it stays with the photo
|
||||
faces = faces[1:]
|
||||
|
||||
#
|
||||
for i, face in enumerate(faces):
|
||||
remove_face_from_identity(identity, face)
|
||||
identity_updates.append(identity)
|
||||
min = None
|
||||
for j, potential in enumerate(identities):
|
||||
if potential == identity:
|
||||
continue
|
||||
distance = findCosineDistanceBaked(face, potential)
|
||||
if distance > MAX_DISTANCE_FROM_CENTROID:
|
||||
continue
|
||||
if min == None or distance < min:
|
||||
face["distance"] = distance
|
||||
face['identity'] = potential
|
||||
face['identityId'] = potential['identityId']
|
||||
|
||||
face_updates.append(face)
|
||||
|
||||
if face['identity'] != None:
|
||||
print(f' {i+1}: {face["id"]} moves from {identity["displayName"]} to:')
|
||||
print(f' {face["identity"]["displayName"]} with distance {face["distance"]}')
|
||||
else:
|
||||
print(f' {i+1}: {face["id"]} needs a new Identity.')
|
||||
|
||||
|
||||
conn = create_connection(db_path)
|
||||
with conn:
|
||||
cur = conn.cursor()
|
||||
for face in face_updates:
|
||||
print(f'Updating face {face["id"]} in DB')
|
||||
if face['identity'] == None:
|
||||
sql = '''
|
||||
UPDATE faces SET identityId=NULL WHERE id=?
|
||||
'''
|
||||
values=(face["id"], )
|
||||
else:
|
||||
sql = '''
|
||||
UPDATE faces SET identityId=? WHERE id=?
|
||||
'''
|
||||
values=(
|
||||
face["identityId"],
|
||||
face["id"]
|
||||
)
|
||||
cur.execute(sql, values)
|
||||
conn.commit()
|
||||
|
||||
@ -172,12 +172,13 @@ function init() {
|
||||
key: 'id',
|
||||
}
|
||||
},
|
||||
|
||||
expertAssignment: {
|
||||
type: Sequelize.BOOLEAN,
|
||||
defaultValue: false
|
||||
classifiedBy: {
|
||||
type: Sequelize.DataTypes.ENUM(
|
||||
'machine',
|
||||
'human',
|
||||
'not-a-face'), /* implies "human"; identityId=NULL */
|
||||
defaultValue: 'machine',
|
||||
},
|
||||
|
||||
lastComparedId: {
|
||||
type: Sequelize.INTEGER,
|
||||
allowNull: true,
|
||||
|
||||
@ -14,12 +14,52 @@ require("../db/photos").then(function(db) {
|
||||
const router = express.Router();
|
||||
const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
|
||||
|
||||
router.put("/:id", function(req, res/*, next*/) {
|
||||
router.put("/:id?", async (req, res/*, next*/) => {
|
||||
console.log(`PUT ${req.url}`);
|
||||
|
||||
if (!req.user.maintainer) {
|
||||
return res.status(401).send("Unauthorized to modify photos.");
|
||||
return res.status(401).json({ message: "Unauthorized to modify photos."});
|
||||
}
|
||||
|
||||
return res.status(400).send("Invalid request");
|
||||
let { id } = req.params, faces = [];
|
||||
if (id && isNaN(+id)) {
|
||||
return res.status(400).json({message: `Invalid id ${id}`});
|
||||
} else {
|
||||
if (id) {
|
||||
faces = [ +id ];
|
||||
} else {
|
||||
faces = req.body.faces;
|
||||
}
|
||||
}
|
||||
if (faces.length === 0) {
|
||||
return res.status(400).json({message: `No faces supplied.`});
|
||||
}
|
||||
const { action } = req.body;
|
||||
console.log(`${action}: ${faces}`);
|
||||
switch (action) {
|
||||
case 'not-a-face':
|
||||
await photoDB.sequelize.query(
|
||||
`UPDATE faces SET classifiedBy='not-a-face',identityId=NULL ` +
|
||||
`WHERE id IN (:faces)`, {
|
||||
replacements: { faces }
|
||||
}
|
||||
);
|
||||
/*
|
||||
faces = await photoDB.sequelize.query(
|
||||
'SELECT * FROM faces WHERE id IN (:faces)', {
|
||||
replacements: { faces },
|
||||
type: photoDB.Sequelize.QueryTypes.SELECT,
|
||||
raw: true
|
||||
}
|
||||
);
|
||||
faces.forEach(face => {
|
||||
face.faceId = face.id;
|
||||
delete face.id;
|
||||
});
|
||||
*/
|
||||
return res.status(200).json(faces);
|
||||
}
|
||||
return res.status(400).json({ message: "Invalid request" });
|
||||
});
|
||||
|
||||
router.delete("/:id", function(req, res/*, next*/) {
|
||||
|
||||
@ -11,6 +11,58 @@ require("../db/photos").then(function(db) {
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.put('/:id', async (req, res) => {
|
||||
console.log(`PUT ${req.url}`)
|
||||
if (!req.user.maintainer) {
|
||||
console.warn(`${req.user.name} attempted to modify photos.`);
|
||||
return res.status(401).send({ message: "Unauthorized to modify photos." });
|
||||
}
|
||||
|
||||
const { id } = req.params;
|
||||
if (!id || isNaN(+id)) {
|
||||
return res.status(400).send({ message: `Invalid identity id ${id}` });
|
||||
}
|
||||
|
||||
const {
|
||||
displayName,
|
||||
firstName,
|
||||
lastName,
|
||||
middleName
|
||||
} = req.body;
|
||||
|
||||
if (displayName === undefined
|
||||
|| firstName === undefined
|
||||
|| lastName === undefined
|
||||
|| middleName === undefined) {
|
||||
return res.status(400).send({ message: `Missing fields` });
|
||||
}
|
||||
|
||||
await photoDB.sequelize.query(
|
||||
'UPDATE identities ' +
|
||||
'SET ' +
|
||||
'displayName=:displayName, ' +
|
||||
'firstName=:firstName, ' +
|
||||
'lastName=:lastName, ' +
|
||||
'middleName=:middleName ' +
|
||||
'WHERE id=:id', {
|
||||
replacements: {
|
||||
displayName,
|
||||
firstName,
|
||||
lastName,
|
||||
middleName,
|
||||
id
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
return res.status(200).json({
|
||||
displayName,
|
||||
firstName,
|
||||
lastName,
|
||||
middleName,
|
||||
id
|
||||
});
|
||||
});
|
||||
|
||||
router.put("/faces/remove/:id", (req, res) => {
|
||||
console.log(`PUT ${req.url}`)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user