Added start of clustering routine

Signed-off-by: James Ketrenos <james_git@ketrenos.com>

server/cluster.py

@@ -0,0 +1,121 @@
+import sys
+import json
+import os
+import piexif
+import sqlite3
+from sqlite3 import Error
+from PIL import Image
+import numpy as np
+from deepface import DeepFace
+from retinaface import RetinaFace
+
+class NpEncoder(json.JSONEncoder):
+  def default(self, obj):
+    if isinstance(obj, np.integer):
+      return int(obj)
+    if isinstance(obj, np.floating):
+      return float(obj)
+    if isinstance(obj, np.ndarray):
+      return obj.tolist()
+        
+model = DeepFace.build_model('ArcFace')
+input_shape = DeepFace.functions.find_input_shape(model)
+
+def create_connection(db_file):
+  """ create a database connection to the SQLite database
+      specified by db_file
+  :param db_file: database file
+  :return: Connection object or None
+  """
+  conn = None
+  try:
+    conn = sqlite3.connect(db_file)
+  except Error as e:
+    print(e)
+
+  return conn
+
+def create_face(conn, face):
+  """
+  Create a new face in the faces table
+  :param conn:
+  :param face:
+  :return: face id
+  """
+  sql = '''
+  INSERT INTO faces(photoId,scanVersion,faceConfidence,top,left,bottom,right)
+  VALUES(?,?,?,?,?,?,?)
+  '''
+  cur = conn.cursor()
+  cur.execute(sql, (
+    face['photoId'], 
+    face['scanVersion'], 
+    face['faceConfidence'],
+    face['top'],
+    face['left'],
+    face['bottom'],
+    face['right']
+  ))
+  conn.commit()
+  return cur.lastrowid
+
+def create_face_descriptor(conn, faceId, descriptor):
+  """
+  Create a new face in the faces table
+  :param conn:
+  :param faceId:
+  :param descriptor:
+  :return: descriptor id
+  """
+  sql = '''
+  INSERT INTO facedescriptors(faceId,model,descriptors)
+  VALUES(?,?,?)
+  '''
+  cur = conn.cursor()
+  cur.execute(sql, (
+    faceId, 
+    descriptor['model'],
+    np.array(descriptor['descriptors'])
+  ))
+  conn.commit()
+  return cur.lastrowid
+
+def update_face_count(conn, photoId, faces):
+  """
+  Update the number of faces that have been matched on a photo
+  :param conn:
+  :param photoId:
+  :param faces:
+  :return: None
+  """
+  sql = '''
+  UPDATE photos SET faces=? WHERE id=?
+  '''
+  cur = conn.cursor()
+  cur.execute(sql, (faces, photoId))
+  conn.commit()
+  return None
+
+base = '/pictures/'
+conn = create_connection('../db/photos.db')
+faces = {}
+identities = {}
+
+with conn:
+  cur = conn.cursor()
+  res = cur.execute('''
+    SELECT faces.id,facedescriptors.descriptors
+    FROM faces
+    LEFT JOIN facedescriptors ON (faces.descriptorId=facedescriptors.id)
+    WHERE faces.identityId IS null
+    ''')
+  for row in res.fetchall():
+    id, descriptors = row
+    if faces[id] is None:
+      face = {}
+      faces[id] = face
+    else:
+      face = faces[id]
+    face['descriptors'] = descriptors
+
+#    update_face_count(conn, photoId, len(faces))

server/db/photos.js

@@ -99,10 +99,22 @@ function init() {
       lastName: Sequelize.STRING,
       firstName: Sequelize.STRING,
       middleName: Sequelize.STRING,
-      name: {
+      displayName: {
         type: Sequelize.STRING,
         allowNull: false
-      }
+      },
+      descriptors: Sequelize.BLOB /* average of all faces mapped to this */
+    }, {
+      timestamps: false
+    });
+
+    const FaceDescriptor = db.sequelize.define('facedescriptor', {
+      id: {
+        type: Sequelize.INTEGER,
+        primaryKey: true,
+        autoIncrement: true
+      },
+      descriptors: Sequelize.BLOB
     }, {
       timestamps: false
     });
@@ -121,6 +133,33 @@ function init() {
           key: 'id',
         }
       },
+      scanVersion: {
+        type: Sequelize.INTEGER,
+        /*
+         * 0 - original scan type
+         * 1 - Retinaface w/ 0.25% face margin
+         */
+        defaultValue: 0
+      },
+
+      top: Sequelize.FLOAT, /* 0..1 * photoId.height */
+      left: Sequelize.FLOAT, /* 0..1 * photoId.width */
+      bottom: Sequelize.FLOAT, /* 0..1 * photoId.height */
+      right: Sequelize.FLOAT, /* 0..1 * photoId.width */
+      faceConfidence: { /* How confident that this is a face? */
+        type: Sequelize.DOUBLE,
+        defaultValue: 0
+      },
+
+      descriptorId: {
+        type: Sequelize.INTEGER,
+        allowNull: true,
+        references: {
+          model: FaceDescriptor,
+          key: 'id',
+        }
+      },
+
       identityId: {
         type: Sequelize.INTEGER,
         allowNull: true,
@@ -129,30 +168,11 @@ function init() {
           key: 'id',
         }
       },
-      scanVersion: {
+      
-        type: Sequelize.INTEGER,
-        /*
-         * 0 - original scan type
-         * 1 - Retinaface w/ 0.25% face increase
-         */
-        defaultValue: 0
-      },
-      identityDistance: { /* How far are markers from identity match? */
-        type: Sequelize.DOUBLE,
-        defaultValue: -1.0
-      },
-      faceConfidence: { /* How confident that this is a face? */
-        type: Sequelize.DOUBLE,
-        defaultValue: 0
-      },
       lastComparedId: {
         type: Sequelize.INTEGER,
         allowNull: true,
-      },
+      }
-      top: Sequelize.FLOAT, /* 0..1 * photoId.height */
-      left: Sequelize.FLOAT, /* 0..1 * photoId.width */
-      bottom: Sequelize.FLOAT, /* 0..1 * photoId.height */
-      right: Sequelize.FLOAT, /* 0..1 * photoId.width */
     }, {
       timestamps: false,
       classMethods: {
@@ -162,42 +182,32 @@ function init() {
       }
     });
 
-    const FaceDescriptor = db.sequelize.define('facedescriptor', {
-      faceId: {
-        type: Sequelize.INTEGER,
-        primaryKey: true,
-        references: {
-          model: Face,
-          key: 'id',
-        }
-      },
-      model: {
-        type: Sequelize.STRING,
-        defaultValue: ""
-      },
-      descriptors: Sequelize.BLOB
-    }, {
-      timestamps: false
-    });
-
     const FaceDistances = db.sequelize.define('facedistance', {
-      face1Id: {
+      descriptor1Id: {
         type: Sequelize.INTEGER,
         allowNull: false,
         references: {
-          model: Face,
+          model: FaceDescriptor,
           key: 'id',
         }
       },
-      face2Id: {
+      descriptor2Id: {
         type: Sequelize.INTEGER,
         allowNull: false,
         references: {
-          model: Face,
+          model: FaceDescriptor,
           key: 'id',
         }
       },
-      distance: {
+      distanceCosine: {
+        type: Sequelize.DOUBLE,
+        defaultValue: 1.0
+      },
+      distanceEuclidian: {
+        type: Sequelize.DOUBLE,
+        defaultValue: 1.0
+      },
+      distanceEuclidianL2: {
         type: Sequelize.DOUBLE,
         defaultValue: 1.0
       }

server/detect.py

@@ -8,6 +8,7 @@ from PIL import Image
 from deepface import DeepFace
 from retinaface import RetinaFace
 import numpy as np
+import cv2
 
 class NpEncoder(json.JSONEncoder):
   def default(self, obj):
@@ -40,33 +41,48 @@ def alignment_procedure(img, left_eye, right_eye):
   """
   dY = right_eye[1] - left_eye[1]
   dX = right_eye[0] - left_eye[0]
-  rotation = -np.atan2(dY, dX)
+  radians = np.arctan2(dY, dX)
-#  cosRotation = np.cos(rotation)
+  rotation = 180 * radians / np.pi
-#  sinRotation = np.sin(rotation)
-#  eyeDistance = np.sqrt(dY * dY + dX * dX)
-#  mid_x = left_eye[0] + 0.5 * dX
-#  mid_y = left_eye[1] + 0.5 * dY
-#  prime_x = mid_x * cosRotation - mid_y * sinRotation
-#  prime_y = mid_y * cosRotation - mid_x * sinRotation
 
+  if True:
     img = img.rotate(
-    angle = np.pi * rotation,
+      angle = rotation,
-    resample=Image.BICUBIC, 
+      resample = Image.BICUBIC, 
-    expand=True)
+      expand = True)
 
   return img
 
-def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True):
+def extract_faces(img, threshold=0.75, model = None, allow_upscaling = True):
-  faces = RetinaFace.detect_faces(img_path = img, threshold = threshold, model = model, allow_upscaling = allow_upscaling)
+  faces = RetinaFace.detect_faces(
-  #faces = DeepFace.detectFace(img_path = img, target_size = (224, 224),    detector_backend = 'retinaface')
+    img_path = img, 
+    threshold = threshold, 
+    model = model, 
+    allow_upscaling = allow_upscaling)
 
   # Re-implementation of 'extract_faces' with the addition of keeping a 
   # copy of the face image for caching on disk
   if type(faces) == dict:
+    print(f'Found {len(faces)} faces')
+    i=1
     for key in faces:
+      print(f'Processing face {i}/{len(faces)}')
+      i+=1
       identity = faces[key]
       facial_area = identity["facial_area"]
+      landmarks = identity["landmarks"]
+      left_eye = landmarks["left_eye"]
+      right_eye = landmarks["right_eye"]
 
+      if False: # Draw the face rectangle and eyes
+        cv2.rectangle(img, 
+          (int(facial_area[0]), int(facial_area[1])),
+          (int(facial_area[2]), int(facial_area[3])),
+          (0, 0, 255), 2)      
+        cv2.circle(img, (int(left_eye[0]), int(left_eye[1])), 5, (255, 0, 0), 2)
+        cv2.circle(img, (int(right_eye[0]), int(right_eye[1])), 5, (0, 255, 0), 2)
+    
+      # Find center of face, then crop to square
+      # of equal width and height
       width = facial_area[2] - facial_area[0]
       height = facial_area[3] - facial_area[1]
       x = facial_area[0] + width * 0.5
@@ -78,34 +94,25 @@ def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True):
       else:
         width = height
 
-      landmarks = identity["landmarks"]
+      #width *= 1.25
-      left_eye = landmarks["left_eye"]
+      #height *= 1.25
-      right_eye = landmarks["right_eye"]
-      nose = landmarks["nose"]
 
-      # translate the landmarks to be centered on array
+      left = max(round(x - width * 0.5), 0)
-      left_eye[0] -= x
+      right = min(round(left + width), img.shape[1])  # Y is 1
-      left_eye[1] -= y
+      top = max(round(y - height * 0.5), 0)
-      right_eye[0] -= x
+      bottom = min(round(top + height), img.shape[0]) # X is 0
-      right_eye[1] -= y
-      nose[0] -= x
-      nose[1] -= y
 
-      width *= 1.25
+      left_eye[0] -= top
-      height *= 1.25
+      left_eye[1] -= left
-
+      right_eye[0] -= top
-      left = max(round(x - width * 0.5), facial_area[0])
+      right_eye[1] -= left
-      right = min(round(left + width), facial_area[2])
-      top = max(round(y - height * 0.5), facial_area[1])
-      bottom = min(round(top + height), facial_area[3])
 
       facial_img = img[top: bottom, left: right]
 
       # Eye order is reversed as the routine does them backwards
-      aligned = RetinaFace.postprocess.alignment_procedure(facial_img, right_eye, left_eye, nose)
+      image = Image.fromarray(facial_img)
-
+      image = alignment_procedure(image, right_eye, left_eye)
-      image = Image.fromarray(aligned)
+      #image = image.resize(size = input_shape, resample = Image.LANCZOS)
-      image = image.resize(size = input_shape, resample = Image.LANCZOS)
       resized = np.asarray(image)
         
       identity['vector'] = DeepFace.represent(
@@ -115,6 +122,7 @@ def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True):
         detector_backend = 'retinaface',
         enforce_detection = False)
 
+      print(img.shape)
       identity["face"] = { 
         'top': facial_area[1] / img.shape[0],
         'left': facial_area[0] / img.shape[1],
@@ -122,7 +130,7 @@ def extract_faces(img, threshold=0.9, model = None, allow_upscaling = True):
         'right': facial_area[2] / img.shape[1]
       }
 
-      identity['image'] = resized #[:, :, ::-1]
+      identity['image'] = Image.fromarray(resized)
 
     return faces
 
@@ -167,7 +175,7 @@ def create_face(conn, face):
   conn.commit()
   return cur.lastrowid
 
-def create_face_descriptor(conn, faceId, descriptor):
+def create_face_descriptor(conn, face):
   """
   Create a new face in the faces table
   :param conn:
@@ -176,15 +184,11 @@ def create_face_descriptor(conn, faceId, descriptor):
   :return: descriptor id
   """
   sql = '''
-  INSERT INTO facedescriptors(faceId,model,descriptors)
+  INSERT INTO facedescriptors(descriptors)
-  VALUES(?,?,?)
+  VALUES(?)
   '''
   cur = conn.cursor()
-  cur.execute(sql, (
+  cur.execute(sql, (np.array(face['vector']),))
-    faceId, 
-    descriptor['model'],
-    np.array(descriptor['descriptors'])
-  ))
   conn.commit()
   return cur.lastrowid
 
@@ -204,23 +208,26 @@ def update_face_count(conn, photoId, faces):
   conn.commit()
   return None
 
-
 base = '/pictures/'
 conn = create_connection('../db/photos.db')
 with conn:
   cur = conn.cursor()
-  for row in cur.execute('''
+  res = cur.execute('''
     SELECT photos.id,photos.faces,albums.path,photos.filename FROM photos
     LEFT JOIN albums ON (albums.id=photos.albumId) 
     WHERE photos.faces=-1
-    '''):
+    ''')
+  rows = res.fetchall()
+  count = len(rows)
+  i=1
+  for row in rows:
     photoId, photoFaces, albumPath, photoFilename = row
     img_path = f'{base}{albumPath}{photoFilename}'
-    print(f'Processing {img_path}')
+    print(f'Processing {i}/{count}: {img_path}')
+    i+=1
     img = Image.open(img_path)
     img = img.convert()
     img = np.asarray(img)
-    print(img.shape)
     faces = extract_faces(img)
     if faces is None:
       update_face_count(conn, photoId, 0)
@@ -228,7 +235,7 @@ with conn:
     print(f'Handling {len(faces)} faces')
     for key in faces:
       face = faces[key]
-      image = Image.fromarray(face['image'])
+      image = face['image']
       
       #face['analysis'] = DeepFace.analyze(img_path = img, actions = ['age', 'gender', 'race', 'emotion'], enforce_detection = False)
       #face['analysis'] = DeepFace.analyze(img, actions = ['emotion'])
@@ -240,6 +247,8 @@ with conn:
       data = {k: face[k] for k in set(list(face.keys())) - set(['image', 'facial_area', 'landmarks'])}
       json_str = json.dumps(data, ensure_ascii=False, indent=2, cls=NpEncoder)
 
+      faceDescriptorId = create_face_descriptor(conn, face)
+
       faceId = create_face(conn, {
         'photoId': photoId,
         'scanVersion': face['version'],
@@ -248,14 +257,10 @@ with conn:
         'left': face['face']['left'],
         'bottom': face['face']['bottom'],
         'right': face['face']['right'],
+        'descriptorId': faceDescriptorId,
       })
 
-      faceDescriptorId = create_face_descriptor(conn, faceId, {
+      path = f'faces/{faceId % 10}'
-        'model': 'RetinaFace',
-        'descriptors': face['vector']
-      })
-
-      path = f'faces/{faceId % 100}'
       try:
         os.mkdir(path)
       except FileExistsError:
@@ -274,6 +279,7 @@ with conn:
       #print(df.head())
 
     update_face_count(conn, photoId, len(faces))
+    exit(0)
 
   #img2_path = sys.argv[2]
 #print("image 1: ", img1_path);