-
-
+
+
Faces: {identity.relatedFaces.length}
{ relatedFacesJSX }
@@ -316,10 +346,18 @@ const Identities = ({ identities, onFaceClick } : IdentitiesProps) => {
return identities.map((identity) => {
const face = identity.relatedFaces[0];
return (
-
+
+
+
);
});
}, [ identities, onFaceClick ]);
@@ -342,7 +380,7 @@ const Button = ({ onClick, children }: any) => {
const App = () => {
const [identities, setIdentities] = useState([]);
const { identityId, faceId } = useParams();
- const [identity, setIdentity] = useState(undefined);
+ const [identity, setIdentity] = useState(undefined);
const [image, setImage] = useState(0);
const { loading, data } = useApi(
`${base}/api/v1/identities`
@@ -359,6 +397,31 @@ const App = () => {
}
};
+ /* If the identity changes, update its entry in the identities list */
+ useEffect(() => {
+ if (!identity || identities.length === 0) {
+ return;
+ }
+ for (let key in identities) {
+ if (identities[key].identityId === identity.identityId) {
+ if (identities[key] !== identity) {
+ /* Inherit all fields from prior identity, while
+ * also keeping the same reference pointer */
+ identities[key] = {
+ ...identity,
+ relatedFaces: identities[key].relatedFaces
+ };
+ /* relatedFaces is a list of references to identity */
+ identity.relatedFaces.forEach(face => {
+ face.identity = identity;
+ });
+ setIdentities([...identities]);
+ }
+ return;
+ }
+ }
+ }, [identity, setIdentities, identities]);
+
useEffect(() => {
if (identityId !== undefined && !isNaN(+identityId)) {
loadIdentity(+identityId);
@@ -367,6 +430,7 @@ const App = () => {
if (faceId !== undefined && !isNaN(+faceId)) {
setImage(+faceId);
}
+ // eslint-disable-next-line
}, []);
useEffect(() => {
@@ -380,7 +444,24 @@ const App = () => {
}
}, [data]);
- const removeSelected = async () => {
+ const removeFacesFromIdentities = (faceIds: number[]) => {
+ if (!identity) {
+ return;
+ }
+ const pre = identity.relatedFaces.length;
+ /* Remove all relatedFaces which are part of the set of removed
+ * faces */
+ identity.relatedFaces = identity.relatedFaces.filter(
+ (face: FaceData) => faceIds.indexOf(face.faceId) === -1);
+ if (pre !== identity.relatedFaces.length) {
+ setIdentity({ ...identity })
+ }
+ }
+
+ const markSelectedIncorrectIdentity = async () => {
+ if (!identity) {
+ return;
+ }
try {
const res = await window.fetch(
`${base}/api/v1/identities/faces/remove/${identity.identityId}`, {
@@ -390,14 +471,25 @@ const App = () => {
});
const data = await res.json();
- const pre = identity.relatedFaces.length;
- /* Remove all relatedFaces which are part of the set of removed
- * faces */
- identity.relatedFaces = identity.relatedFaces.filter(
- (face: FaceData) => data.faces.indexOf(face.faceId) === -1);
- if (pre !== identity.relatedFaces.length) {
- setIdentity({...identity})
- }
+ removeFacesFromIdentities(data.faces);
+ } catch (error) {
+ console.error(error);
+ }
+ };
+
+ const markSelectedNotFace = async () => {
+ try {
+ const res = await window.fetch(
+ `${base}/api/v1/faces`, {
+ method: 'PUT',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ action: 'not-a-face',
+ faces: selected
+ })
+ });
+ const data = await res.json();
+ removeFacesFromIdentities(data);
} catch (error) {
console.error(error);
}
@@ -409,17 +501,14 @@ const App = () => {
return;
}
const identityId = face.identity.identityId;
- console.log(face.identity);
- const identitiesEl = document.querySelector('.Identities');
- if (!identitiesEl) {
- return;
- }
- const faceIdentity = identitiesEl.querySelector(
- `[data-identity-id="${identityId}"]`);
- if (!faceIdentity) {
- return;
- }
- faceIdentity.scrollIntoView()
+ const faceId = face.faceId;
+ console.log(`onFaceClick`, { faceId, identityId});
+ const faces = [
+ ...document.querySelectorAll(`.Identities [data-identity-id="${identityId}"]`),
+ ...document.querySelectorAll(`.Cluster [data-face-id="${faceId}"]`)];
+ faces.forEach((el: any) => {
+ el.scrollIntoView();
+ });
};
const identitiesOnFaceClick = (e: any, face: FaceData) => {
@@ -434,18 +523,21 @@ const App = () => {
autoSaveId="persistence" direction="horizontal">
{loading && Loading...
}
- {!loading && identity !== 0 &&
+ {!loading && identity !== undefined &&
}
- {!loading && identity === 0 &&
+ {!loading && identity === undefined &&
Select identity to edit
}
- { selected.length !== 0 && }
+ { selected.length !== 0 && <>
+
+
+ }
@@ -453,9 +545,12 @@ const App = () => {
{image === 0 &&
Select image to view
}
{image !== 0 &&
}
-
- { !loading && }
+
+
+ { !loading && }
+
+
);
diff --git a/ketrface/cluster.py b/ketrface/cluster.py
index 05f5bbd..87aee24 100644
--- a/ketrface/cluster.py
+++ b/ketrface/cluster.py
@@ -27,19 +27,6 @@ if html_base == "/":
MAX_CLUSTER_DISTANCE = 0.14 # Used to merge clusters
MAX_DISTANCE_FROM_CENTROID = 0.14 # Used to prune outliers
-# TODO
-# Switch to using DBSCAN
-#
-# Thoughts for determining number of clusters to try and target...
-#
-# Augment DBSCAN to rule out identity matching for the same face
-# appearing more than once in a photo
-#
-# NOTE: This means twins or reflections won't both identify in the
-# same photo -- those faces would then identify as a second face pairing
-# which could merge with a cluster, but can not be used to match
-
-
def gen_html(identities):
for identity in identities:
@@ -79,41 +66,6 @@ def update_cluster_averages(identities):
average, average)))
return identities
-def load_faces(db_path = db_path):
- print(f'Connecting to database: {db_path}')
- conn = create_connection(db_path)
- faces = []
- with conn:
- print('Querying faces')
- cur = conn.cursor()
- res = cur.execute('''
- SELECT faces.id,facedescriptors.descriptors,faces.faceConfidence,faces.photoId,faces.focus
- FROM faces
- INNER JOIN photos ON (photos.duplicate == 0 OR photos.duplicate IS NULL)
- JOIN facedescriptors ON (faces.descriptorId=facedescriptors.id)
- WHERE faces.identityId IS null AND faces.faceConfidence>0.99
- AND faces.photoId=photos.id
- ''')
- for row in res.fetchall():
- id, descriptors, confidence, photoId, focus = row
- if focus is None:
- focus = 100 # Assume full focus if focus not set
- face = {
- 'id': id,
- 'type': 'face',
- 'confidence': confidence,
- 'distance': 0,
- 'photoId': photoId,
- 'descriptors': np.frombuffer(descriptors),
- 'cluster': Undefined,
- 'focus': focus
- }
- face['faces'] = [ face ]
- face['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
- face['descriptors'], face['descriptors'])))
- faces.append(face)
- return faces
-
def update_distances(identities,
prune = False,
maxDistance = MAX_DISTANCE_FROM_CENTROID):
@@ -169,7 +121,7 @@ def build_straglers(faces):
return noise + undefined
print('Loading faces from database')
-faces = load_faces()
+faces = load_faces(db_path = db_path)
minPts = max(len(faces) / 500, 5)
eps = 0.185
@@ -267,41 +219,6 @@ redirect_on(os.path.join(html_path, 'auto-clusters.html'))
gen_html(reduced)
redirect_off()
-def create_identity(conn, identity):
- """
- Create a new identity in the identities table
- :param conn:
- :param identity:
- :return: identity id
- """
- sql = '''
- INSERT INTO identities(descriptors,displayName)
- VALUES(?,?)
- '''
- cur = conn.cursor()
- cur.execute(sql, (
- np.array(identity['descriptors']),
- f'cluster-{identity["id"]}'
- ))
- conn.commit()
- return cur.lastrowid
-
-def update_face_identity(conn, faceId, identityId = None):
- """
- Update the identity associated with this face
- :param conn:
- :param faceId:
- :param identityId:
- :return: None
- """
- sql = '''
- UPDATE faces SET identityId=? WHERE id=?
- '''
- cur = conn.cursor()
- cur.execute(sql, (identityId, faceId))
- conn.commit()
- return None
-
print(f'Connecting to database: {db_path}')
conn = create_connection(db_path)
with conn:
diff --git a/ketrface/ketrface/db.py b/ketrface/ketrface/db.py
index 57e65d7..b3e8a96 100644
--- a/ketrface/ketrface/db.py
+++ b/ketrface/ketrface/db.py
@@ -76,3 +76,83 @@ def update_face_count(conn, photoId, faces):
cur.execute(sql, (faces, photoId))
conn.commit()
return None
+
+
+import sys
+import json
+import os
+import piexif
+import sqlite3
+from sqlite3 import Error
+from PIL import Image
+import numpy as np
+
+def load_faces(db_path ):
+ print(f'Connecting to database: {db_path}')
+ conn = create_connection(db_path)
+ faces = []
+ with conn:
+ print('Querying faces')
+ cur = conn.cursor()
+ res = cur.execute('''
+ SELECT faces.id,facedescriptors.descriptors,faces.faceConfidence,faces.photoId,faces.focus
+ FROM faces
+ INNER JOIN photos ON (photos.duplicate == 0 OR photos.duplicate IS NULL)
+ JOIN facedescriptors ON (faces.descriptorId=facedescriptors.id)
+ WHERE faces.identityId IS null AND faces.faceConfidence>0.99
+ AND faces.photoId=photos.id
+ ''')
+ for row in res.fetchall():
+ id, descriptors, confidence, photoId, focus = row
+ if focus is None:
+ focus = 100 # Assume full focus if focus not set
+ face = {
+ 'id': id,
+ 'type': 'face',
+ 'confidence': confidence,
+ 'distance': 0,
+ 'photoId': photoId,
+ 'descriptors': np.frombuffer(descriptors),
+ 'cluster': 0, # Undefined from dbscan.py
+ 'focus': focus
+ }
+ face['faces'] = [ face ]
+ face['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+ face['descriptors'], face['descriptors'])))
+ faces.append(face)
+ return faces
+
+def create_identity(conn, identity):
+ """
+ Create a new identity in the identities table
+ :param conn:
+ :param identity:
+ :return: identity id
+ """
+ sql = '''
+ INSERT INTO identities(descriptors,displayName)
+ VALUES(?,?)
+ '''
+ cur = conn.cursor()
+ cur.execute(sql, (
+ np.array(identity['descriptors']),
+ f'cluster-{identity["id"]}'
+ ))
+ conn.commit()
+ return cur.lastrowid
+
+def update_face_identity(conn, faceId, identityId = None):
+ """
+ Update the identity associated with this face
+ :param conn:
+ :param faceId:
+ :param identityId:
+ :return: None
+ """
+ sql = '''
+ UPDATE faces SET identityId=? WHERE id=?
+ '''
+ cur = conn.cursor()
+ cur.execute(sql, (identityId, faceId))
+ conn.commit()
+ return None
diff --git a/ketrface/split.py b/ketrface/split.py
new file mode 100644
index 0000000..a88d22a
--- /dev/null
+++ b/ketrface/split.py
@@ -0,0 +1,255 @@
+import sys
+import json
+import os
+import piexif
+import sqlite3
+from sqlite3 import Error
+from PIL import Image
+import numpy as np
+
+import functools
+
+from ketrface.util import *
+from ketrface.dbscan import *
+from ketrface.db import *
+from ketrface.config import *
+
+MAX_DISTANCE_FROM_CENTROID = 0.14 # Used to prune outliers
+
+config = read_config()
+
+html_path = merge_config_path(config['path'], 'frontend')
+pictures_path = merge_config_path(config['path'], config['picturesPath'])
+faces_path = merge_config_path(config['path'], config['facesPath'])
+db_path = merge_config_path(config['path'], config["db"]["photos"]["host"])
+html_base = config['basePath']
+if html_base == "/":
+ html_base = "."
+
+def update_cluster_averages(identities):
+ for identity in identities:
+ average = []
+ for face in identity['faces']:
+ if len(average) == 0:
+ average = face['descriptors']
+ else:
+ average = np.add(average, face['descriptors'])
+ average = np.divide(average, len(identity['faces']))
+ identity['descriptors'] = average
+ identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+ average, average)))
+ return identities
+
+
+def sort_identities(identities):
+ identities.sort(reverse = True, key = lambda x: len(x['faces']))
+ for identity in identities:
+ identity['faces'].sort(reverse = False, key = lambda x: x['distance'])
+
+def cluster_sort(A, B):
+ diff = A['cluster'] - B['cluster']
+ if diff > 0:
+ return 1
+ elif diff < 0:
+ return -1
+ diff = A['confidence'] - B['confidence']
+ if diff > 0:
+ return 1
+ elif diff < 0:
+ return -1
+ return 0
+
+def load_identities(db_path):
+ conn = create_connection(db_path)
+ identities = []
+ with conn:
+ cur = conn.cursor()
+ res = cur.execute('''
+ SELECT
+ identities.id as identityId,
+ identities.displayName as displayName,
+ identities.descriptors as descriptors,
+ COUNT(faces.id) AS faceCount
+ FROM identities
+ JOIN faces ON faces.identityId=identities.id
+ GROUP BY identities.id
+ ''')
+ for row in res.fetchall():
+ identityId, displayName, descriptors, faceCount = row
+ identity = {
+ 'identityId': identityId,
+ 'displayName': displayName,
+ 'descriptors': np.frombuffer(descriptors),
+ 'faceCount': faceCount,
+ 'updated': False
+ }
+ # Pre-bake computations for cosine distance
+ identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+ identity['descriptors'], identity['descriptors'])))
+ identities.append(identity)
+ return identities
+
+def find_identity(identities, identityId):
+ for element in identities:
+ if element['identityId'] == identityId:
+ return element
+ raise Exception(f'Identity {identityId} missing')
+
+def load_doppelganger_photos(db_path):
+ conn = create_connection(db_path)
+ photos = {}
+ with conn:
+ cur = conn.cursor()
+ res = cur.execute('''
+ SELECT
+ f1.identityId AS identityId,
+ photos.id AS photoId,
+ f1.id AS f1_id,
+ f2.id AS f2_id,
+ f1_descriptors.descriptors AS f1_descriptors,
+ f2_descriptors.descriptors AS f2_descriptors
+ FROM faces AS f1
+ INNER JOIN faces AS f2 ON (
+ f2.identityId=f1.identityId AND f1.photoId=f2.photoId and f1_id!=f2_id)
+ INNER JOIN photos ON (
+ photos.duplicate == 0 OR photos.duplicate IS NULL)
+ INNER JOIN facedescriptors AS f1_descriptors ON (
+ f1.descriptorId=f1_descriptors.id)
+ INNER JOIN facedescriptors AS f2_descriptors ON (
+ f2.descriptorId=f2_descriptors.id)
+ WHERE f1.identityId IS NOT NULL AND f1.photoId=photos.id
+ ORDER BY photos.id,f1.identityId
+ ''')
+ for row in res.fetchall():
+ identityId, photoId, f1_id, f2_id,f1_descriptors, f2_descriptors = row
+ face1 = {
+ 'id': f1_id,
+ 'type': 'face',
+ 'distance': 0,
+ 'descriptors': np.frombuffer(f1_descriptors),
+ 'cluster': identityId, # Undefined from dbscan.py
+ }
+ face1['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+ face1['descriptors'], face1['descriptors'])))
+
+ face2 = {
+ 'id': f2_id,
+ 'type': 'face',
+ 'distance': 0,
+ 'descriptors': np.frombuffer(f2_descriptors),
+ 'cluster': identityId, # Undefined from dbscan.py
+ }
+ face2['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+ face2['descriptors'], face2['descriptors'])))
+
+ if photoId not in photos:
+ photos[photoId] = {
+ 'photoId': photoId,
+ 'dopplegangers': {}
+ }
+ if identityId not in photos[photoId]['dopplegangers']:
+ photos[photoId]['dopplegangers'][identityId] = {
+ 'identity': None,
+ 'faces': []
+ }
+
+ faceList = photos[photoId]['dopplegangers'][identityId]['faces']
+ for face in [ face1, face2 ]:
+ match = False
+ for key in faceList:
+ if face['id'] == key['id']:
+ match = True
+ break
+ if not match:
+ faceList.append(face)
+
+ return photos
+
+def remove_face_from_identity(identity, face):
+ identity['updated'] = True
+ average = identity['descriptors']
+ average = np.dot(average, identity['faceCount'])
+ average = np.subtract(average, face['descriptors'])
+ identity['faceCount'] -= 1
+ average = np.divide(average, identity['faceCount'])
+ identity['descriptors'] = average
+ identity['sqrtsummul'] = np.sqrt(np.sum(np.multiply(
+ average, average)))
+
+ face['identity'] = None
+ face['identityId'] = -1
+ face['distance'] = 0
+
+print('Loading identities from database...')
+identities = load_identities(db_path)
+print(f'{len(identities)} identities loaded.')
+print('Loading dopplegangers from database...')
+photos = load_doppelganger_photos(db_path)
+print(f'{len(photos)} photos with dopplegangers loaded.')
+print('Binding dopplegangers to identities...')
+
+face_updates = []
+identity_updates = []
+for photoId in photos:
+ photo = photos[photoId]
+ print(f'Processing photo {photoId}...')
+ for identityId in photo['dopplegangers']:
+ if photo['dopplegangers'][identityId]['identity'] == None:
+ photo['dopplegangers'][identityId]['identity'] = find_identity(identities, identityId)
+ faces = photo['dopplegangers'][identityId]['faces']
+ identity = photo['dopplegangers'][identityId]['identity']
+
+ for face in faces:
+ face['identity'] = identity
+ face['distance'] = findCosineDistanceBaked(face, identity)
+ faces.sort(reverse = False, key = lambda x: x['distance'])
+
+ # First face closest to identity -- it stays with the photo
+ faces = faces[1:]
+
+ #
+ for i, face in enumerate(faces):
+ remove_face_from_identity(identity, face)
+ identity_updates.append(identity)
+ min = None
+ for j, potential in enumerate(identities):
+ if potential == identity:
+ continue
+ distance = findCosineDistanceBaked(face, potential)
+ if distance > MAX_DISTANCE_FROM_CENTROID:
+ continue
+ if min == None or distance < min:
+ face["distance"] = distance
+ face['identity'] = potential
+ face['identityId'] = potential['identityId']
+
+ face_updates.append(face)
+
+ if face['identity'] != None:
+ print(f' {i+1}: {face["id"]} moves from {identity["displayName"]} to:')
+ print(f' {face["identity"]["displayName"]} with distance {face["distance"]}')
+ else:
+ print(f' {i+1}: {face["id"]} needs a new Identity.')
+
+
+conn = create_connection(db_path)
+with conn:
+ cur = conn.cursor()
+ for face in face_updates:
+ print(f'Updating face {face["id"]} in DB')
+ if face['identity'] == None:
+ sql = '''
+ UPDATE faces SET identityId=NULL WHERE id=?
+ '''
+ values=(face["id"], )
+ else:
+ sql = '''
+ UPDATE faces SET identityId=? WHERE id=?
+ '''
+ values=(
+ face["identityId"],
+ face["id"]
+ )
+ cur.execute(sql, values)
+ conn.commit()
+
diff --git a/server/db/photos.js b/server/db/photos.js
index f3cf139..d7e962d 100755
--- a/server/db/photos.js
+++ b/server/db/photos.js
@@ -172,12 +172,13 @@ function init() {
key: 'id',
}
},
-
- expertAssignment: {
- type: Sequelize.BOOLEAN,
- defaultValue: false
+ classifiedBy: {
+ type: Sequelize.DataTypes.ENUM(
+ 'machine',
+ 'human',
+ 'not-a-face'), /* implies "human"; identityId=NULL */
+ defaultValue: 'machine',
},
-
lastComparedId: {
type: Sequelize.INTEGER,
allowNull: true,
diff --git a/server/routes/faces.js b/server/routes/faces.js
index e0149c4..41ccce8 100755
--- a/server/routes/faces.js
+++ b/server/routes/faces.js
@@ -14,12 +14,52 @@ require("../db/photos").then(function(db) {
const router = express.Router();
const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
-router.put("/:id", function(req, res/*, next*/) {
+router.put("/:id?", async (req, res/*, next*/) => {
+ console.log(`PUT ${req.url}`);
+
if (!req.user.maintainer) {
- return res.status(401).send("Unauthorized to modify photos.");
+ return res.status(401).json({ message: "Unauthorized to modify photos."});
}
- return res.status(400).send("Invalid request");
+ let { id } = req.params, faces = [];
+ if (id && isNaN(+id)) {
+ return res.status(400).json({message: `Invalid id ${id}`});
+ } else {
+ if (id) {
+ faces = [ +id ];
+ } else {
+ faces = req.body.faces;
+ }
+ }
+ if (faces.length === 0) {
+ return res.status(400).json({message: `No faces supplied.`});
+ }
+ const { action } = req.body;
+ console.log(`${action}: ${faces}`);
+ switch (action) {
+ case 'not-a-face':
+ await photoDB.sequelize.query(
+ `UPDATE faces SET classifiedBy='not-a-face',identityId=NULL ` +
+ `WHERE id IN (:faces)`, {
+ replacements: { faces }
+ }
+ );
+ /*
+ faces = await photoDB.sequelize.query(
+ 'SELECT * FROM faces WHERE id IN (:faces)', {
+ replacements: { faces },
+ type: photoDB.Sequelize.QueryTypes.SELECT,
+ raw: true
+ }
+ );
+ faces.forEach(face => {
+ face.faceId = face.id;
+ delete face.id;
+ });
+ */
+ return res.status(200).json(faces);
+ }
+ return res.status(400).json({ message: "Invalid request" });
});
router.delete("/:id", function(req, res/*, next*/) {
diff --git a/server/routes/identities.js b/server/routes/identities.js
index 4dd6708..8a4193a 100755
--- a/server/routes/identities.js
+++ b/server/routes/identities.js
@@ -11,6 +11,58 @@ require("../db/photos").then(function(db) {
const router = express.Router();
+router.put('/:id', async (req, res) => {
+ console.log(`PUT ${req.url}`)
+ if (!req.user.maintainer) {
+ console.warn(`${req.user.name} attempted to modify photos.`);
+ return res.status(401).send({ message: "Unauthorized to modify photos." });
+ }
+
+ const { id } = req.params;
+ if (!id || isNaN(+id)) {
+ return res.status(400).send({ message: `Invalid identity id ${id}` });
+ }
+
+ const {
+ displayName,
+ firstName,
+ lastName,
+ middleName
+ } = req.body;
+
+ if (displayName === undefined
+ || firstName === undefined
+ || lastName === undefined
+ || middleName === undefined) {
+ return res.status(400).send({ message: `Missing fields` });
+ }
+
+ await photoDB.sequelize.query(
+ 'UPDATE identities ' +
+ 'SET ' +
+ 'displayName=:displayName, ' +
+ 'firstName=:firstName, ' +
+ 'lastName=:lastName, ' +
+ 'middleName=:middleName ' +
+ 'WHERE id=:id', {
+ replacements: {
+ displayName,
+ firstName,
+ lastName,
+ middleName,
+ id
+ }
+ }
+ );
+
+ return res.status(200).json({
+ displayName,
+ firstName,
+ lastName,
+ middleName,
+ id
+ });
+});
router.put("/faces/remove/:id", (req, res) => {
console.log(`PUT ${req.url}`)
