jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Added start of clustering routine

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2023-01-05 17:49:54 -08:00
parent 836b27ac54
commit 7a960c5f1f
3 changed files with 242 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
server/cluster.py Normal file
View File

@ -0,0 +1,121 @@
import sys
import json
import os
import piexif
import sqlite3
from sqlite3 import Error
from PIL import Image
import numpy as np
from deepface import DeepFace
from retinaface import RetinaFace
class NpEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, np.integer):
return int(obj)
if isinstance(obj, np.floating):
return float(obj)
if isinstance(obj, np.ndarray):
return obj.tolist()
model = DeepFace.build_model('ArcFace')
input_shape = DeepFace.functions.find_input_shape(model)
def create_connection(db_file):
""" create a database connection to the SQLite database
specified by db_file
:param db_file: database file
:return: Connection object or None
"""
conn = None
try:
conn = sqlite3.connect(db_file)
except Error as e:
print(e)
return conn
def create_face(conn, face):
"""
Create a new face in the faces table
:param conn:
:param face:
:return: face id
"""
sql = '''
INSERT INTO faces(photoId,scanVersion,faceConfidence,top,left,bottom,right)
VALUES(?,?,?,?,?,?,?)
'''
cur = conn.cursor()
cur.execute(sql, (
face['photoId'],
face['scanVersion'],
face['faceConfidence'],
face['top'],
face['left'],
face['bottom'],
face['right']
))
conn.commit()
return cur.lastrowid
def create_face_descriptor(conn, faceId, descriptor):
"""
Create a new face in the faces table
:param conn:
:param faceId:
:param descriptor:
:return: descriptor id
"""
sql = '''
INSERT INTO facedescriptors(faceId,model,descriptors)
VALUES(?,?,?)
'''
cur = conn.cursor()
cur.execute(sql, (
faceId,
descriptor['model'],
np.array(descriptor['descriptors'])
))
conn.commit()
return cur.lastrowid
def update_face_count(conn, photoId, faces):
"""
Update the number of faces that have been matched on a photo
:param conn:
:param photoId:
:param faces:
:return: None
"""
sql = '''
UPDATE photos SET faces=? WHERE id=?
'''
cur = conn.cursor()
cur.execute(sql, (faces, photoId))
conn.commit()
return None
base = '/pictures/'
conn = create_connection('../db/photos.db')
faces = {}
identities = {}
with conn:
cur = conn.cursor()
res = cur.execute('''
SELECT faces.id,facedescriptors.descriptors
FROM faces
LEFT JOIN facedescriptors ON (faces.descriptorId=facedescriptors.id)
WHERE faces.identityId IS null
''')
for row in res.fetchall():
id, descriptors = row
if faces[id] is None:
face = {}
faces[id] = face
else:
face = faces[id]
face['descriptors'] = descriptors
# update_face_count(conn, photoId, len(faces))

102
server/db/photos.js
View File

@ -99,10 +99,22 @@ function init() {
lastName: Sequelize.STRING,
firstName: Sequelize.STRING,
middleName: Sequelize.STRING,
name: {
displayName: {
type: Sequelize.STRING,
allowNull: false
}
},
descriptors: Sequelize.BLOB /* average of all faces mapped to this */
}, {
timestamps: false
});
const FaceDescriptor = db.sequelize.define('facedescriptor', {
id: {
type: Sequelize.INTEGER,
primaryKey: true,
autoIncrement: true
},
descriptors: Sequelize.BLOB
}, {
timestamps: false
});
@ -121,6 +133,33 @@ function init() {
key: 'id',
}
},
scanVersion: {
type: Sequelize.INTEGER,
/*
* 0 - original scan type
* 1 - Retinaface w/ 0.25% face margin
*/
defaultValue: 0
},
top: Sequelize.FLOAT, /* 0..1 * photoId.height */
left: Sequelize.FLOAT, /* 0..1 * photoId.width */
bottom: Sequelize.FLOAT, /* 0..1 * photoId.height */
right: Sequelize.FLOAT, /* 0..1 * photoId.width */
faceConfidence: { /* How confident that this is a face? */
type: Sequelize.DOUBLE,
defaultValue: 0
},
descriptorId: {
type: Sequelize.INTEGER,
allowNull: true,
references: {
model: FaceDescriptor,
key: 'id',
}
},
identityId: {
type: Sequelize.INTEGER,
allowNull: true,
@ -129,30 +168,11 @@ function init() {
key: 'id',
}
},
scanVersion: {
type: Sequelize.INTEGER,
/*
* 0 - original scan type
* 1 - Retinaface w/ 0.25% face increase
*/
defaultValue: 0
},
identityDistance: { /* How far are markers from identity match? */
type: Sequelize.DOUBLE,
defaultValue: -1.0
},
faceConfidence: { /* How confident that this is a face? */
type: Sequelize.DOUBLE,
defaultValue: 0
},
lastComparedId: {
type: Sequelize.INTEGER,
allowNull: true,
},
top: Sequelize.FLOAT, /* 0..1 * photoId.height */
left: Sequelize.FLOAT, /* 0..1 * photoId.width */
bottom: Sequelize.FLOAT, /* 0..1 * photoId.height */
right: Sequelize.FLOAT, /* 0..1 * photoId.width */
}
}, {
timestamps: false,
classMethods: {
@ -162,42 +182,32 @@ function init() {
}
});
const FaceDescriptor = db.sequelize.define('facedescriptor', {
faceId: {
type: Sequelize.INTEGER,
primaryKey: true,
references: {
model: Face,
key: 'id',
}
},
model: {
type: Sequelize.STRING,
defaultValue: ""
},
descriptors: Sequelize.BLOB
}, {
timestamps: false
});
const FaceDistances = db.sequelize.define('facedistance', {
face1Id: {
descriptor1Id: {
type: Sequelize.INTEGER,
allowNull: false,
references: {
model: Face,
model: FaceDescriptor,
key: 'id',
}
},
face2Id: {
descriptor2Id: {
type: Sequelize.INTEGER,
allowNull: false,
references: {
model: Face,
model: FaceDescriptor,
key: 'id',
}
},
distance: {
distanceCosine: {
type: Sequelize.DOUBLE,
defaultValue: 1.0
},
distanceEuclidian: {
type: Sequelize.DOUBLE,
defaultValue: 1.0
},
distanceEuclidianL2: {
type: Sequelize.DOUBLE,
defaultValue: 1.0
}

124
server/detect.py
View File

@ -8,6 +8,7 @@ from PIL import Image
from deepface import DeepFace
from retinaface import RetinaFace
import numpy as np
import cv2
class NpEncoder(json.JSONEncoder):
def default(self, obj):
@ -40,33 +41,48 @@ def alignment_procedure(img, left_eye, right_eye):
"""
dY = right_eye[1] - left_eye[1]
dX = right_eye[0] - left_eye[0]
rotation = -np.atan2(dY, dX)
# cosRotation = np.cos(rotation)
# sinRotation = np.sin(rotation)
# eyeDistance = np.sqrt(dY * dY + dX * dX)
# mid_x = left_eye[0] + 0.5 * dX
# mid_y = left_eye[1] + 0.5 * dY
# prime_x = mid_x * cosRotation - mid_y * sinRotation
# prime_y = mid_y * cosRotation - mid_x * sinRotation
radians = np.arctan2(dY, dX)
rotation = 180 * radians / np.pi
if True:
img = img.rotate(
angle = rotation,
resample = Image.BICUBIC,
expand = True)
img = img.rotate(
angle = np.pi * rotation,
resample=Image.BICUBIC,
expand=True)
return img
def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True):
faces = RetinaFace.detect_faces(img_path = img, threshold = threshold, model = model, allow_upscaling = allow_upscaling)
#faces = DeepFace.detectFace(img_path = img, target_size = (224, 224), detector_backend = 'retinaface')
def extract_faces(img, threshold=0.75, model = None, allow_upscaling = True):
faces = RetinaFace.detect_faces(
img_path = img,
threshold = threshold,
model = model,
allow_upscaling = allow_upscaling)
# Re-implementation of 'extract_faces' with the addition of keeping a
# copy of the face image for caching on disk
if type(faces) == dict:
print(f'Found {len(faces)} faces')
i=1
for key in faces:
print(f'Processing face {i}/{len(faces)}')
i+=1
identity = faces[key]
facial_area = identity["facial_area"]
landmarks = identity["landmarks"]
left_eye = landmarks["left_eye"]
right_eye = landmarks["right_eye"]
if False: # Draw the face rectangle and eyes
cv2.rectangle(img,
(int(facial_area[0]), int(facial_area[1])),
(int(facial_area[2]), int(facial_area[3])),
(0, 0, 255), 2)
cv2.circle(img, (int(left_eye[0]), int(left_eye[1])), 5, (255, 0, 0), 2)
cv2.circle(img, (int(right_eye[0]), int(right_eye[1])), 5, (0, 255, 0), 2)
# Find center of face, then crop to square
# of equal width and height
width = facial_area[2] - facial_area[0]
height = facial_area[3] - facial_area[1]
x = facial_area[0] + width * 0.5
@ -78,34 +94,25 @@ def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True):
else:
width = height
landmarks = identity["landmarks"]
left_eye = landmarks["left_eye"]
right_eye = landmarks["right_eye"]
nose = landmarks["nose"]
#width *= 1.25
#height *= 1.25
# translate the landmarks to be centered on array
left_eye[0] -= x
left_eye[1] -= y
right_eye[0] -= x
right_eye[1] -= y
nose[0] -= x
nose[1] -= y
left = max(round(x - width * 0.5), 0)
right = min(round(left + width), img.shape[1]) # Y is 1
top = max(round(y - height * 0.5), 0)
bottom = min(round(top + height), img.shape[0]) # X is 0
width *= 1.25
height *= 1.25
left = max(round(x - width * 0.5), facial_area[0])
right = min(round(left + width), facial_area[2])
top = max(round(y - height * 0.5), facial_area[1])
bottom = min(round(top + height), facial_area[3])
left_eye[0] -= top
left_eye[1] -= left
right_eye[0] -= top
right_eye[1] -= left
facial_img = img[top: bottom, left: right]
# Eye order is reversed as the routine does them backwards
aligned = RetinaFace.postprocess.alignment_procedure(facial_img, right_eye, left_eye, nose)
image = Image.fromarray(aligned)
image = image.resize(size = input_shape, resample = Image.LANCZOS)
image = Image.fromarray(facial_img)
image = alignment_procedure(image, right_eye, left_eye)
#image = image.resize(size = input_shape, resample = Image.LANCZOS)
resized = np.asarray(image)
identity['vector'] = DeepFace.represent(
@ -115,6 +122,7 @@ def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True):
detector_backend = 'retinaface',
enforce_detection = False)
print(img.shape)
identity["face"] = {
'top': facial_area[1] / img.shape[0],
'left': facial_area[0] / img.shape[1],
@ -122,7 +130,7 @@ def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True):
'right': facial_area[2] / img.shape[1]
}
identity['image'] = resized #[:, :, ::-1]
identity['image'] = Image.fromarray(resized)
return faces
@ -167,7 +175,7 @@ def create_face(conn, face):
conn.commit()
return cur.lastrowid
def create_face_descriptor(conn, faceId, descriptor):
def create_face_descriptor(conn, face):
"""
Create a new face in the faces table
:param conn:
@ -176,15 +184,11 @@ def create_face_descriptor(conn, faceId, descriptor):
:return: descriptor id
"""
sql = '''
INSERT INTO facedescriptors(faceId,model,descriptors)
VALUES(?,?,?)
INSERT INTO facedescriptors(descriptors)
VALUES(?)
'''
cur = conn.cursor()
cur.execute(sql, (
faceId,
descriptor['model'],
np.array(descriptor['descriptors'])
))
cur.execute(sql, (np.array(face['vector']),))
conn.commit()
return cur.lastrowid
@ -204,23 +208,26 @@ def update_face_count(conn, photoId, faces):
conn.commit()
return None
base = '/pictures/'
conn = create_connection('../db/photos.db')
with conn:
cur = conn.cursor()
for row in cur.execute('''
res = cur.execute('''
SELECT photos.id,photos.faces,albums.path,photos.filename FROM photos
LEFT JOIN albums ON (albums.id=photos.albumId)
WHERE photos.faces=-1
'''):
''')
rows = res.fetchall()
count = len(rows)
i=1
for row in rows:
photoId, photoFaces, albumPath, photoFilename = row
img_path = f'{base}{albumPath}{photoFilename}'
print(f'Processing {img_path}')
print(f'Processing {i}/{count}: {img_path}')
i+=1
img = Image.open(img_path)
img = img.convert()
img = np.asarray(img)
print(img.shape)
faces = extract_faces(img)
if faces is None:
update_face_count(conn, photoId, 0)
@ -228,7 +235,7 @@ with conn:
print(f'Handling {len(faces)} faces')
for key in faces:
face = faces[key]
image = Image.fromarray(face['image'])
image = face['image']
#face['analysis'] = DeepFace.analyze(img_path = img, actions = ['age', 'gender', 'race', 'emotion'], enforce_detection = False)
#face['analysis'] = DeepFace.analyze(img, actions = ['emotion'])
@ -240,6 +247,8 @@ with conn:
data = {k: face[k] for k in set(list(face.keys())) - set(['image', 'facial_area', 'landmarks'])}
json_str = json.dumps(data, ensure_ascii=False, indent=2, cls=NpEncoder)
faceDescriptorId = create_face_descriptor(conn, face)
faceId = create_face(conn, {
'photoId': photoId,
'scanVersion': face['version'],
@ -248,14 +257,10 @@ with conn:
'left': face['face']['left'],
'bottom': face['face']['bottom'],
'right': face['face']['right'],
'descriptorId': faceDescriptorId,
})
faceDescriptorId = create_face_descriptor(conn, faceId, {
'model': 'RetinaFace',
'descriptors': face['vector']
})
path = f'faces/{faceId % 100}'
path = f'faces/{faceId % 10}'
try:
os.mkdir(path)
except FileExistsError:
@ -274,6 +279,7 @@ with conn:
#print(df.head())
update_face_count(conn, photoId, len(faces))
exit(0)
#img2_path = sys.argv[2]
#print("image 1: ", img1_path);