From 7a960c5f1f7d902ce19c0cfa29ff6b2000897d00 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Thu, 5 Jan 2023 17:49:54 -0800 Subject: [PATCH] Added start of clustering routine Signed-off-by: James Ketrenos --- server/cluster.py | 121 ++++++++++++++++++++++++++++++++++++++++++ server/db/photos.js | 102 ++++++++++++++++++++---------------- server/detect.py | 124 +++++++++++++++++++++++--------------------- 3 files changed, 242 insertions(+), 105 deletions(-) create mode 100644 server/cluster.py diff --git a/server/cluster.py b/server/cluster.py new file mode 100644 index 0000000..d028d32 --- /dev/null +++ b/server/cluster.py @@ -0,0 +1,121 @@ +import sys +import json +import os +import piexif +import sqlite3 +from sqlite3 import Error +from PIL import Image +import numpy as np +from deepface import DeepFace +from retinaface import RetinaFace + +class NpEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, np.integer): + return int(obj) + if isinstance(obj, np.floating): + return float(obj) + if isinstance(obj, np.ndarray): + return obj.tolist() + +model = DeepFace.build_model('ArcFace') +input_shape = DeepFace.functions.find_input_shape(model) + +def create_connection(db_file): + """ create a database connection to the SQLite database + specified by db_file + :param db_file: database file + :return: Connection object or None + """ + conn = None + try: + conn = sqlite3.connect(db_file) + except Error as e: + print(e) + + return conn + +def create_face(conn, face): + """ + Create a new face in the faces table + :param conn: + :param face: + :return: face id + """ + sql = ''' + INSERT INTO faces(photoId,scanVersion,faceConfidence,top,left,bottom,right) + VALUES(?,?,?,?,?,?,?) + ''' + cur = conn.cursor() + cur.execute(sql, ( + face['photoId'], + face['scanVersion'], + face['faceConfidence'], + face['top'], + face['left'], + face['bottom'], + face['right'] + )) + conn.commit() + return cur.lastrowid + +def create_face_descriptor(conn, faceId, descriptor): + """ + Create a new face in the faces table + :param conn: + :param faceId: + :param descriptor: + :return: descriptor id + """ + sql = ''' + INSERT INTO facedescriptors(faceId,model,descriptors) + VALUES(?,?,?) + ''' + cur = conn.cursor() + cur.execute(sql, ( + faceId, + descriptor['model'], + np.array(descriptor['descriptors']) + )) + conn.commit() + return cur.lastrowid + +def update_face_count(conn, photoId, faces): + """ + Update the number of faces that have been matched on a photo + :param conn: + :param photoId: + :param faces: + :return: None + """ + sql = ''' + UPDATE photos SET faces=? WHERE id=? + ''' + cur = conn.cursor() + cur.execute(sql, (faces, photoId)) + conn.commit() + return None + +base = '/pictures/' +conn = create_connection('../db/photos.db') +faces = {} +identities = {} + +with conn: + cur = conn.cursor() + res = cur.execute(''' + SELECT faces.id,facedescriptors.descriptors + FROM faces + LEFT JOIN facedescriptors ON (faces.descriptorId=facedescriptors.id) + WHERE faces.identityId IS null + ''') + for row in res.fetchall(): + id, descriptors = row + if faces[id] is None: + face = {} + faces[id] = face + else: + face = faces[id] + face['descriptors'] = descriptors + +# update_face_count(conn, photoId, len(faces)) diff --git a/server/db/photos.js b/server/db/photos.js index f53a26e..2a1f13a 100755 --- a/server/db/photos.js +++ b/server/db/photos.js @@ -99,10 +99,22 @@ function init() { lastName: Sequelize.STRING, firstName: Sequelize.STRING, middleName: Sequelize.STRING, - name: { + displayName: { type: Sequelize.STRING, allowNull: false - } + }, + descriptors: Sequelize.BLOB /* average of all faces mapped to this */ + }, { + timestamps: false + }); + + const FaceDescriptor = db.sequelize.define('facedescriptor', { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true + }, + descriptors: Sequelize.BLOB }, { timestamps: false }); @@ -121,6 +133,33 @@ function init() { key: 'id', } }, + scanVersion: { + type: Sequelize.INTEGER, + /* + * 0 - original scan type + * 1 - Retinaface w/ 0.25% face margin + */ + defaultValue: 0 + }, + + top: Sequelize.FLOAT, /* 0..1 * photoId.height */ + left: Sequelize.FLOAT, /* 0..1 * photoId.width */ + bottom: Sequelize.FLOAT, /* 0..1 * photoId.height */ + right: Sequelize.FLOAT, /* 0..1 * photoId.width */ + faceConfidence: { /* How confident that this is a face? */ + type: Sequelize.DOUBLE, + defaultValue: 0 + }, + + descriptorId: { + type: Sequelize.INTEGER, + allowNull: true, + references: { + model: FaceDescriptor, + key: 'id', + } + }, + identityId: { type: Sequelize.INTEGER, allowNull: true, @@ -129,30 +168,11 @@ function init() { key: 'id', } }, - scanVersion: { - type: Sequelize.INTEGER, - /* - * 0 - original scan type - * 1 - Retinaface w/ 0.25% face increase - */ - defaultValue: 0 - }, - identityDistance: { /* How far are markers from identity match? */ - type: Sequelize.DOUBLE, - defaultValue: -1.0 - }, - faceConfidence: { /* How confident that this is a face? */ - type: Sequelize.DOUBLE, - defaultValue: 0 - }, + lastComparedId: { type: Sequelize.INTEGER, allowNull: true, - }, - top: Sequelize.FLOAT, /* 0..1 * photoId.height */ - left: Sequelize.FLOAT, /* 0..1 * photoId.width */ - bottom: Sequelize.FLOAT, /* 0..1 * photoId.height */ - right: Sequelize.FLOAT, /* 0..1 * photoId.width */ + } }, { timestamps: false, classMethods: { @@ -162,42 +182,32 @@ function init() { } }); - const FaceDescriptor = db.sequelize.define('facedescriptor', { - faceId: { - type: Sequelize.INTEGER, - primaryKey: true, - references: { - model: Face, - key: 'id', - } - }, - model: { - type: Sequelize.STRING, - defaultValue: "" - }, - descriptors: Sequelize.BLOB - }, { - timestamps: false - }); - const FaceDistances = db.sequelize.define('facedistance', { - face1Id: { + descriptor1Id: { type: Sequelize.INTEGER, allowNull: false, references: { - model: Face, + model: FaceDescriptor, key: 'id', } }, - face2Id: { + descriptor2Id: { type: Sequelize.INTEGER, allowNull: false, references: { - model: Face, + model: FaceDescriptor, key: 'id', } }, - distance: { + distanceCosine: { + type: Sequelize.DOUBLE, + defaultValue: 1.0 + }, + distanceEuclidian: { + type: Sequelize.DOUBLE, + defaultValue: 1.0 + }, + distanceEuclidianL2: { type: Sequelize.DOUBLE, defaultValue: 1.0 } diff --git a/server/detect.py b/server/detect.py index b0d7b8b..406c3d5 100644 --- a/server/detect.py +++ b/server/detect.py @@ -8,6 +8,7 @@ from PIL import Image from deepface import DeepFace from retinaface import RetinaFace import numpy as np +import cv2 class NpEncoder(json.JSONEncoder): def default(self, obj): @@ -40,33 +41,48 @@ def alignment_procedure(img, left_eye, right_eye): """ dY = right_eye[1] - left_eye[1] dX = right_eye[0] - left_eye[0] - rotation = -np.atan2(dY, dX) -# cosRotation = np.cos(rotation) -# sinRotation = np.sin(rotation) -# eyeDistance = np.sqrt(dY * dY + dX * dX) -# mid_x = left_eye[0] + 0.5 * dX -# mid_y = left_eye[1] + 0.5 * dY -# prime_x = mid_x * cosRotation - mid_y * sinRotation -# prime_y = mid_y * cosRotation - mid_x * sinRotation + radians = np.arctan2(dY, dX) + rotation = 180 * radians / np.pi + + if True: + img = img.rotate( + angle = rotation, + resample = Image.BICUBIC, + expand = True) - img = img.rotate( - angle = np.pi * rotation, - resample=Image.BICUBIC, - expand=True) - return img -def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True): - faces = RetinaFace.detect_faces(img_path = img, threshold = threshold, model = model, allow_upscaling = allow_upscaling) - #faces = DeepFace.detectFace(img_path = img, target_size = (224, 224), detector_backend = 'retinaface') +def extract_faces(img, threshold=0.75, model = None, allow_upscaling = True): + faces = RetinaFace.detect_faces( + img_path = img, + threshold = threshold, + model = model, + allow_upscaling = allow_upscaling) # Re-implementation of 'extract_faces' with the addition of keeping a # copy of the face image for caching on disk if type(faces) == dict: + print(f'Found {len(faces)} faces') + i=1 for key in faces: + print(f'Processing face {i}/{len(faces)}') + i+=1 identity = faces[key] facial_area = identity["facial_area"] + landmarks = identity["landmarks"] + left_eye = landmarks["left_eye"] + right_eye = landmarks["right_eye"] + if False: # Draw the face rectangle and eyes + cv2.rectangle(img, + (int(facial_area[0]), int(facial_area[1])), + (int(facial_area[2]), int(facial_area[3])), + (0, 0, 255), 2) + cv2.circle(img, (int(left_eye[0]), int(left_eye[1])), 5, (255, 0, 0), 2) + cv2.circle(img, (int(right_eye[0]), int(right_eye[1])), 5, (0, 255, 0), 2) + + # Find center of face, then crop to square + # of equal width and height width = facial_area[2] - facial_area[0] height = facial_area[3] - facial_area[1] x = facial_area[0] + width * 0.5 @@ -78,34 +94,25 @@ def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True): else: width = height - landmarks = identity["landmarks"] - left_eye = landmarks["left_eye"] - right_eye = landmarks["right_eye"] - nose = landmarks["nose"] + #width *= 1.25 + #height *= 1.25 - # translate the landmarks to be centered on array - left_eye[0] -= x - left_eye[1] -= y - right_eye[0] -= x - right_eye[1] -= y - nose[0] -= x - nose[1] -= y + left = max(round(x - width * 0.5), 0) + right = min(round(left + width), img.shape[1]) # Y is 1 + top = max(round(y - height * 0.5), 0) + bottom = min(round(top + height), img.shape[0]) # X is 0 - width *= 1.25 - height *= 1.25 - - left = max(round(x - width * 0.5), facial_area[0]) - right = min(round(left + width), facial_area[2]) - top = max(round(y - height * 0.5), facial_area[1]) - bottom = min(round(top + height), facial_area[3]) + left_eye[0] -= top + left_eye[1] -= left + right_eye[0] -= top + right_eye[1] -= left facial_img = img[top: bottom, left: right] # Eye order is reversed as the routine does them backwards - aligned = RetinaFace.postprocess.alignment_procedure(facial_img, right_eye, left_eye, nose) - - image = Image.fromarray(aligned) - image = image.resize(size = input_shape, resample = Image.LANCZOS) + image = Image.fromarray(facial_img) + image = alignment_procedure(image, right_eye, left_eye) + #image = image.resize(size = input_shape, resample = Image.LANCZOS) resized = np.asarray(image) identity['vector'] = DeepFace.represent( @@ -115,6 +122,7 @@ def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True): detector_backend = 'retinaface', enforce_detection = False) + print(img.shape) identity["face"] = { 'top': facial_area[1] / img.shape[0], 'left': facial_area[0] / img.shape[1], @@ -122,7 +130,7 @@ def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True): 'right': facial_area[2] / img.shape[1] } - identity['image'] = resized #[:, :, ::-1] + identity['image'] = Image.fromarray(resized) return faces @@ -167,7 +175,7 @@ def create_face(conn, face): conn.commit() return cur.lastrowid -def create_face_descriptor(conn, faceId, descriptor): +def create_face_descriptor(conn, face): """ Create a new face in the faces table :param conn: @@ -176,15 +184,11 @@ def create_face_descriptor(conn, faceId, descriptor): :return: descriptor id """ sql = ''' - INSERT INTO facedescriptors(faceId,model,descriptors) - VALUES(?,?,?) + INSERT INTO facedescriptors(descriptors) + VALUES(?) ''' cur = conn.cursor() - cur.execute(sql, ( - faceId, - descriptor['model'], - np.array(descriptor['descriptors']) - )) + cur.execute(sql, (np.array(face['vector']),)) conn.commit() return cur.lastrowid @@ -204,23 +208,26 @@ def update_face_count(conn, photoId, faces): conn.commit() return None - base = '/pictures/' conn = create_connection('../db/photos.db') with conn: cur = conn.cursor() - for row in cur.execute(''' + res = cur.execute(''' SELECT photos.id,photos.faces,albums.path,photos.filename FROM photos LEFT JOIN albums ON (albums.id=photos.albumId) WHERE photos.faces=-1 - '''): + ''') + rows = res.fetchall() + count = len(rows) + i=1 + for row in rows: photoId, photoFaces, albumPath, photoFilename = row img_path = f'{base}{albumPath}{photoFilename}' - print(f'Processing {img_path}') + print(f'Processing {i}/{count}: {img_path}') + i+=1 img = Image.open(img_path) img = img.convert() img = np.asarray(img) - print(img.shape) faces = extract_faces(img) if faces is None: update_face_count(conn, photoId, 0) @@ -228,7 +235,7 @@ with conn: print(f'Handling {len(faces)} faces') for key in faces: face = faces[key] - image = Image.fromarray(face['image']) + image = face['image'] #face['analysis'] = DeepFace.analyze(img_path = img, actions = ['age', 'gender', 'race', 'emotion'], enforce_detection = False) #face['analysis'] = DeepFace.analyze(img, actions = ['emotion']) @@ -240,6 +247,8 @@ with conn: data = {k: face[k] for k in set(list(face.keys())) - set(['image', 'facial_area', 'landmarks'])} json_str = json.dumps(data, ensure_ascii=False, indent=2, cls=NpEncoder) + faceDescriptorId = create_face_descriptor(conn, face) + faceId = create_face(conn, { 'photoId': photoId, 'scanVersion': face['version'], @@ -248,14 +257,10 @@ with conn: 'left': face['face']['left'], 'bottom': face['face']['bottom'], 'right': face['face']['right'], + 'descriptorId': faceDescriptorId, }) - faceDescriptorId = create_face_descriptor(conn, faceId, { - 'model': 'RetinaFace', - 'descriptors': face['vector'] - }) - - path = f'faces/{faceId % 100}' + path = f'faces/{faceId % 10}' try: os.mkdir(path) except FileExistsError: @@ -274,6 +279,7 @@ with conn: #print(df.head()) update_face_count(conn, photoId, len(faces)) + exit(0) #img2_path = sys.argv[2] #print("image 1: ", img1_path);