From d019af070d54c8e676c3b8726ed4011629de7707 Mon Sep 17 00:00:00 2001
From: James Ketrenos <james_git@ketrenos.com>
Date: Fri, 13 Jan 2023 14:09:46 -0800
Subject: [PATCH] Updating slideshow and identity editor

Signed-off-by: James Ketrenos <james_git@ketrenos.com>
---
 docker-compose.yml          |   1 +
 frontend/identities.html    |   3 +-
 frontend/slideshow.html     | 342 +++++++++++++++++++++---------------
 ketrface/detect.py          |  26 +--
 ketrface/ketrface/db.py     |   1 -
 server/routes/identities.js |  15 +-
 server/routes/photos.js     |  88 +++++-----
 7 files changed, 274 insertions(+), 202 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 9d7af9b..9a284bc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -18,4 +18,5 @@ services:
       - ${PWD}/config/local.json:/website/config/local.json
       - /opt/ketrface/models:/root/.deepface
 #      - ${PWD}:/website
+      - ${PWD}/frontend:/website/frontend
       - ${PWD}/server:/website/server
diff --git a/frontend/identities.html b/frontend/identities.html
index ae8176d..1d0e3e2 100755
--- a/frontend/identities.html
+++ b/frontend/identities.html
@@ -18,7 +18,8 @@ function createFace(faceId, photoId, selectable) {
   div.classList.add("face");
   div.setAttribute("photo-id", photoId);
   div.setAttribute("face-id", faceId);
-  div.style.backgroundImage = "url(face-data/" + (faceId % 100) + "/" + faceId + "-original.png)";
+  const dir = String(faceId % 100).padStart(2, '0')
+  div.style.backgroundImage = `url(faces/${dir}/${faceId}.jpg)`;
   div.addEventListener("click", (event) => {
     if (event.shiftKey) { /* identities */
       let faceId = parseInt(event.currentTarget.getAttribute("face-id"));
diff --git a/frontend/slideshow.html b/frontend/slideshow.html
index be3026a..5b9fb8b 100755
--- a/frontend/slideshow.html
+++ b/frontend/slideshow.html
@@ -77,39 +77,73 @@ body {
   <div id="loading"></div>    
 </div>
 <script>
-var base, placeholder, info, photos = [], photoIndex;
 
-function shuffle(array) {
-  var index = array.length, tmp, random;
+let base, 
+  placeholder, 
+  info, 
+  photos = [], 
+  photoIndex = -1;
 
-  while (index) {
-    random = Math.floor(Math.random() * index);
-    index--;
+let mode, filter;
 
-    // And swap it with the current element.
-    tmp = array[index];
-    array[index] = array[random];
-    array[random] = tmp;
-  }
-
-  return array;
-}
-
-var countdown = 15;
-
-const days = [ "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" ],
-      months = [ "January", "February", "March", "April", "May", "June",
-            "July", "August", "September", "October", "November", "December" ];
+const days = [
+  "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday",
+  "Saturday"
+];
+const months = [
+  "January", "February", "March", "April", "May", "June",
+  "July", "August", "September", "October", "November", "December"
+];
 
 let activeFaces = [];
 
-function makeFaceBoxes() {
+let paused = false,
+    tap = 0;
+let countdown = 15;
+let scheduled = false;
+
+
+const onClick = (e) => {
+  const now = new Date().getTime();
+  if (tap && (now - tap < 300)) {
+    toggleFullscreen();
+    tap = 0;
+  } else {
+    tap = new Date().getTime();
+  }
+};
+
+const shuffle = (arr) => {
+  var index = arr.length, tmp, random;
+  while (index) {
+    random = Math.floor(Math.random() * index);
+    index--;
+    tmp = arr[index];
+    arr[index] = arr[random];
+    arr[random] = tmp;
+  }
+  return arr;
+};
+
+const faceClick = (event, face) => {
+  console.log(face);
+  face.relatedFaces.forEach((photo) => {
+    window.open(base + photo.path);
+  });
+  event.preventDefault = true;
+  event.stopImmediatePropagation();
+  event.stopPropagation();
+  return false;
+};
+
+const makeFaceBoxes = () => {
   Array.prototype.forEach.call(document.querySelectorAll('.face'), (el) => {
     el.parentElement.removeChild(el);
   });
 
-console.log("Not showing face boxes");
-return;
+  if (activeFaces.length === 0) {
+    return;
+  }
 
   const el = document.getElementById("photo"),
     photo = photos[photoIndex];
@@ -129,7 +163,6 @@ return;
     offsetTop = 0;
   }
 
-  
   activeFaces.forEach((face) => {
     const box = document.createElement("div");
     box.classList.add("face");
@@ -138,20 +171,11 @@ return;
     box.style.top = offsetTop + Math.floor(face.top * height)  + "%";
     box.style.width = Math.floor((face.right - face.left) * width)  + "%";
     box.style.height = Math.floor((face.bottom - face.top) * height)  + "%";
-    box.addEventListener("click", (event) => {
-      console.log(face);
-      face.relatedPhotos.forEach((photo) => {
-        window.open(base + photo.path);
-      });
-      event.preventDefault = true;
-      event.stopImmediatePropagation();
-      event.stopPropagation();
-      return false;
-    });
+    box.addEventListener("click", (e) => { return faceClick(e, face); });
   });
-}
+};
 
-function loadPhoto(index) {
+const loadPhoto = (index) => {
   const photo = photos[index],
     xml = new XMLHttpRequest(),
     url = base + photo.path + "thumbs/scaled/" + photo.filename,
@@ -169,23 +193,29 @@ function loadPhoto(index) {
     }
   };
 
-  xml.onload = function(event) {
+  xml.onload = async (event) => {
     info.textContent = 
       days[taken.getDay()] + ", " +
       months[taken.getMonth()] + " " +
       taken.getDate() + " " +
       taken.getFullYear();
-    document.getElementById("photo").style.backgroundImage = "url(" + encodeURI(url).replace(/\(/g, '%28').replace(/\)/g, '%29') + ")";
+    activeFaces = [];
+    makeFaceBoxes();
+    document.getElementById("photo").style.backgroundImage = 
+    `url(${encodeURI(url).replace(/\(/g, '%28').replace(/\)/g, '%29')})`;
     countdown = 15;
     tick();
-    window.fetch("api/v1/photos/faces/" + photo.id).then(res => res.json()).then((faces) => {
+
+    try {
+      const res = await window.fetch("api/v1/photos/faces/" + photo.id);
+      const faces = await res.json();
       activeFaces = faces;
       makeFaceBoxes(photo);
-    }).catch(function(error) {
+    } catch (error) {
       console.error(error);
       info.textContent += "Unable to obtain face information :(";
-    });
-  }
+    }
+  };
 
   xml.onerror = function(event) {
     info.textContent = "Error loading photo. Trying next photo.";
@@ -196,14 +226,77 @@ function loadPhoto(index) {
   xml.send();
 }
 
-function nextPhoto() {
-  photoIndex = (photoIndex + 1) % photos.length;
-  loadPhoto(photoIndex);
+const prevPhoto = async () => {
+  if (photoIndex > 0) {
+    photoIndex--;
+    loadPhoto(photoIndex);
+    return;
+  }
+
+  if (mode !== 'random/') {
+    photoIndex = photos.length;
+    loadPhoto(photoIndex);
+    return;
+  }
+
+  try {
+    const res = await window.fetch(
+      `api/v1/photos/${mode}${filter.replace(/ +/g, "%20")}`);
+    const data = await res.json();
+    if (data && data.items) {
+      info.textContent = photos.length + " photos found. Shuffling.";
+      photos = shuffle(data.items);
+      photoIndex = (photoIndex + 1) % photos.length;
+      loadPhoto(photoIndex);
+    } else if (data) {
+      photos.push(data);
+      photoIndex = photos.length - 1;
+      loadPhoto(photoIndex);
+    } else {
+      info.textContent = "No photos found for " + filter + ".";
+    }
+  } catch(error) {
+    console.error(error);
+    info.textContent = "Unable to fetch " + mode + "=" + filter + " :(";
+  }
 }
 
-var scheduled = false;
+const nextPhoto = async () => {
+  if (photoIndex < photos.length - 1) {
+    photoIndex++;
+    loadPhoto(photoIndex);
+    return;
+  }
 
-function tick() {
+  if (mode !== 'random/') {
+    photoIndex = 0;
+    loadPhoto(photoIndex);
+    return;
+  }
+
+  try {
+    const res = await window.fetch(
+      `api/v1/photos/${mode}${filter.replace(/ +/g, "%20")}`);
+    const data = await res.json();
+    if (data && data.items) {
+      info.textContent = photos.length + " photos found. Shuffling.";
+      photos = shuffle(data.items);
+      photoIndex = 0;
+      loadPhoto(photoIndex);
+    } else if (data) {
+      photos.push(data);
+      photoIndex = photos.length - 1;
+      loadPhoto(photoIndex);
+    } else {
+      info.textContent = "No photos found for " + filter + ".";
+    }
+  } catch (error) {
+    console.error(error);
+    info.textContent = "Unable to fetch " + mode + "=" + filter + " :(";
+  }
+}
+
+const tick = () => {
   if (scheduled) {
     clearTimeout(scheduled);
   }
@@ -219,16 +312,16 @@ function tick() {
     countdown = 15;
     nextPhoto();
   }
-}
+};
 
-function schedule() {
+const schedule = () => {
   if (scheduled) {
     clearTimeout(scheduled);
   }
   tick();
-}
+};
 
-function toggleFullscreen() {
+const toggleFullscreen = () => {
   if (!document.fullscreenElement) {
     document.documentElement.requestFullscreen();
   } else {
@@ -236,104 +329,75 @@ function toggleFullscreen() {
       document.exitFullscreen(); 
     }
   }
-}
+};
 
-var paused = false,
-  tap = 0;
+const onKeyDown = (event) => {
+  switch (event.keyCode) {
+  case 32: /* space */
+    paused = !paused;
+    if (!paused) {
+      tick();
+    } else {
+      document.getElementById("loading").textContent = "||";
+      clearTimeout(scheduled);
+      scheduled = null;
+    }
+    return;
+
+  case 37: /* left */
+    prevPhoto();
+    return;
+
+  case 39: /* right */
+    nextPhoto();
+    return;
+
+  case 13: /* enter */
+    toggleFullscreen();
+    return;
+  }
+};
 
 document.addEventListener("DOMContentLoaded", function() {
-    var tmp = document.querySelector("base");
-    if (tmp) {
-        base = new URL(tmp.href).pathname.replace(/\/$/, "") + "/"; /* Make sure there is a trailing slash */
-    } else {
-        base = "/";
+  var tmp = document.querySelector("base");
+  if (tmp) {
+    /* Make sure there is a trailing slash */
+    base = new URL(tmp.href).pathname.replace(/\/$/, "") + "/"; 
+  } else {
+      base = "/";
+  }
+
+  var timeout = 0;
+  window.addEventListener("resize", (event) => {
+    if (timeout) {
+      window.clearTimeout(timeout);
     }
+    timeout = window.setTimeout(makeFaceBoxes, 250);
+  });
 
-    var timeout = 0;
-    window.addEventListener("resize", (event) => {
-      if (timeout) {
-        window.clearTimeout(timeout);
-      }
-      timeout = window.setTimeout(makeFaceBoxes, 250);
-    });
+  document.addEventListener("click", onClick);
+  document.addEventListener("keydown", onKeyDown);
 
-    document.addEventListener("click", function(event) {
-      toggleFullscreen();
-      var now = new Date().getTime();
-      if (tap && (now - tap < 300)) {
-        toggleFullscreen();
-        tap = 0;
-      } else {
-        tap = new Date().getTime();
-      }
-    });
+  info = document.getElementById("info");
 
-    document.addEventListener("keydown", function(event) {
-      switch (event.keyCode) {
-      case 32: /* space */
-        paused = !paused;
-        if (!paused) {
-          tick();
-        } else {
-          document.getElementById("loading").textContent = "||";
-          clearTimeout(scheduled);
-          scheduled = null;
-        }
-        return;
-
-      case 37: /* left */
-        if (photoIndex == 0) {
-          photoIndex = photos.length;
-        }
-        photoIndex--;
-        loadPhoto(photoIndex);
-        return;
-
-      case 39: /* right */
-        photoIndex = (photoIndex + 1) % photos.length;
-        loadPhoto(photoIndex);
-        return;
-
-      case 13: /* enter */
-        toggleFullscreen();
-        return;
-      }
-
-      console.log(event.keyCode);
-    });
-
-    info = document.getElementById("info");
-
-    /* Trim off everything up to and including the ? (if its there) */
-    var parts = window.location.href.match(/^.*\?(.*)$/),
-      mode = "holiday",
-      filter = "";
-    if (parts) {
-      parts = parts[1].split("=");
-      if (parts.length == 1) {
-        filter = parts[0];
-      } else {
-        mode = parts[0];
-        filter = parts[1];
-      }
+  /* Trim off everything up to and including the ? (if its there) */
+  var parts = window.location.href.match(/^.*\?(.*)$/);
+  if (parts) {
+    parts = parts[1].split("=");
+    if (parts.length == 1) {
+      mode = "holiday/";
+      filter = parts[0];
     } else {
-      filter = "memorial day";
+      mode = parts[0].replace(/\/*$/, '/');
+      filter = parts[1];
     }
+    mode = mode
+  } else {
+    mode = "random/"
+    filter = "";
+  }
 
-    window.fetch("api/v1/photos/" + mode + "/" + filter.replace(/ +/g, "%20"))
-    .then(res => res.json()).then(function(data) {
-      if (data.items.length) {
-        info.textContent = photos.length + " photos found. Shuffling.";
-        photos = shuffle(data.items);
-        photoIndex = -1;
-        nextPhoto();
-      } else {
-        info.textContent = "No photos found for " + filter + ".";
-      }
-    }).catch(function(error) {
-      console.error(error);
-      info.textContent = "Unable to fetch " + mode + "=" + filter + " :(";
-    });
+  nextPhoto();
 });
 </script>
 </body>
diff --git a/ketrface/detect.py b/ketrface/detect.py
index b696ec5..605be33 100644
--- a/ketrface/detect.py
+++ b/ketrface/detect.py
@@ -3,12 +3,15 @@ import json
 import os
 import piexif
 
+import argparse
+
 from PIL import Image, ImageOps
 from deepface import DeepFace
 from deepface.detectors import FaceDetector
 from retinaface import RetinaFace
 import numpy as np
 import cv2
+
 from ketrface.util import *
 from ketrface.db import *
 from ketrface.config import *
@@ -25,7 +28,8 @@ model_name = 'VGG-Face'    # 'ArcFace'
 detector_backend = 'mtcnn' # 'retinaface'
 model = DeepFace.build_model(model_name)
 
-# Derived from https://github.com/serengil/deepface/blob/master/deepface/detectors/MtcnnWrapper.py
+# Derived from
+# https://github.com/serengil/deepface/blob/master/deepface/detectors/MtcnnWrapper.py
 # Add parameters to MTCNN
 from mtcnn import MTCNN
 face_detector = MTCNN(min_face_size = 30)
@@ -63,7 +67,9 @@ def variance_of_laplacian(image):
 	# measure, which is simply the variance of the Laplacian
 	return cv2.Laplacian(image, cv2.CV_64F).var()
 
-def extract_faces(img, threshold=0.95, allow_upscaling = True, focus_threshold = 100):
+def extract_faces(
+  img, threshold=0.95, allow_upscaling = True, focus_threshold = 100):
+
   if detector_backend == 'retinaface':
     faces = RetinaFace.detect_faces(
       img_path = img, 
@@ -103,8 +109,6 @@ def extract_faces(img, threshold=0.95, allow_upscaling = True, focus_threshold =
           
         }
 
-  to_drop = []
-
   # Re-implementation of 'extract_faces' with the addition of keeping a 
   # copy of the face image for caching on disk
   for k, key in enumerate(faces):
@@ -182,12 +186,15 @@ def extract_faces(img, threshold=0.95, allow_upscaling = True, focus_threshold =
 
     identity['image'] = Image.fromarray(resized)
 
-#  for key in to_drop:
-#    faces.pop(key)
- 
   return faces
 
 
+parser = argparse.ArgumentParser(description = 'Detect faces in images.')
+parser.add_argument('photos', metavar='PHOTO', type=int, nargs='*',
+                    help='PHOTO ID to scan (default: all unscanned photos)')
+args = parser.parse_args()
+print(args)
+
 base = '/pictures/'
 conn = create_connection('../db/photos.db')
 with conn:
@@ -222,11 +229,6 @@ with conn:
       image = face['image']
       print(f'Writing face {j+1}/{len(faces)}')
       
-      #face['analysis'] = DeepFace.analyze(img_path = img, actions = ['age', 'gender', 'race', 'emotion'], enforce_detection = False)
-      #face['analysis'] = DeepFace.analyze(img, actions = ['emotion'])
-
-      # TODO: Add additional meta-data allowing back referencing to original
-      # photo
       face['version'] = 1 # version 1 doesn't add much...
 
       data = {k: face[k] for k in set(list(face.keys())) - set(['image', 'facial_area', 'landmarks'])}
diff --git a/ketrface/ketrface/db.py b/ketrface/ketrface/db.py
index 39f0788..57e65d7 100644
--- a/ketrface/ketrface/db.py
+++ b/ketrface/ketrface/db.py
@@ -45,7 +45,6 @@ def create_face(conn, face):
   conn.commit()
   return cur.lastrowid
 
-
 def create_face_descriptor(conn, face):
   """
   Create a new face in the faces table
diff --git a/server/routes/identities.js b/server/routes/identities.js
index 4d8f7e4..ebde173 100755
--- a/server/routes/identities.js
+++ b/server/routes/identities.js
@@ -95,7 +95,7 @@ function euclideanDistance(a, b) {
   let A = bufferToFloat32Array(a);
   let B = bufferToFloat32Array(b);
   let sum = 0;
-  for (let i = 0; i < 128; i++) {
+  for (let i = 0; i < A.length; i++) {
     let delta = A[i] - B[i];
     sum += delta * delta;
   }
@@ -118,14 +118,13 @@ router.get("/:id?", (req, res) => {
     "identities.*," +
     "GROUP_CONCAT(faces.id) AS relatedFaceIds," +
     "GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds," +
-    "GROUP_CONCAT(faces.identityDistance) AS relatedIdentityDistances " +
+    "GROUP_CONCAT(facedescriptors.descriptors) AS relatedIdentityDescriptors " +
     "FROM identities " + 
+    "INNER JOIN facedescriptors ON facedescriptors.id=faces.descriptorId " +
     "INNER JOIN faces ON identities.id=faces.identityId " +
     filter +
     "GROUP BY identities.id", {
-    replacements: {
-      id: id
-    },
+    replacements: { id },
     type: photoDB.Sequelize.QueryTypes.SELECT,
     raw: true
   }).then((identities) => {
@@ -139,10 +138,14 @@ router.get("/:id?", (req, res) => {
       delete identity.relatedFaceIds;
       delete identity.relatedFacePhotoIds;
       identity.relatedFaces = relatedFaces.map((faceId, index) => {
+        const distance = euclideanDistance(
+          relatedIdentityDistances[index],
+          identity.descriptors
+        );
         return {
           faceId: faceId,
           photoId: relatedFacePhotos[index],
-          distance: parseFloat(relatedIdentityDistances[index] !== undefined ? relatedIdentityDistances[index] : -1)
+          distance
         };
       });
     });
diff --git a/server/routes/photos.js b/server/routes/photos.js
index 16a9e20..366a4d8 100755
--- a/server/routes/photos.js
+++ b/server/routes/photos.js
@@ -951,7 +951,7 @@ router.get("/faces/:id", (req, res) => {
   });
 });
 
-router.get("/random/:id?", (req, res) => {
+router.get("/random/:id?", async (req, res) => {
   let id = parseInt(req.params.id),
     filter = "";
 
@@ -959,54 +959,56 @@ router.get("/random/:id?", (req, res) => {
     console.log("GET /random/" + id);
     filter = "AND id=:id";
   } else {
-    filter = "AND faces>0";
+    console.log("GET /random/");
+    filter = "";//AND faces>0";
     id = undefined;
   }
 
-  return photoDB.sequelize.query("SELECT id,duplicate FROM photos WHERE deleted=0 " + filter, {
-    replacements: {
-      id: id
-    },
+  /* If the requested ID is a duplicate, we need to find the original
+   * photo ID */
+  const results = await photoDB.sequelize.query(
+    "SELECT id,duplicate FROM photos WHERE deleted=0 " + filter, {
+    replacements: { id },
     type: photoDB.Sequelize.QueryTypes.SELECT,
     raw: true
-  }).then((results) => {
-    if (!results.length) {
-      return [];
-    }
-    if (id) {
-      if (results[0].duplicate) {
-        id = results[0].duplicate;
-      }
-    } else {
-      id = results[Math.floor(Math.random() * results.length)].id;
-    }
-
-    return photoDB.sequelize.query(
-      "SELECT photos.*,albums.path AS path FROM photos " +
-      "INNER JOIN albums ON albums.id=photos.albumId " +
-      "WHERE photos.duplicate=0 AND photos.deleted=0 AND photos.scanned NOT NULL AND photos.id=:id", {
-      replacements: {
-        id: id,
-      },
-      type: photoDB.Sequelize.QueryTypes.SELECT,
-      raw: true
-    });
-  }).then(function(photos) {
-    if (!photos.length) {
-      return res.status(404).send({ message: id + " not found." });
-    }
-    const photo = photos[0];
-    for (var key in photo) {
-      if (photo[key] instanceof Date) {
-        photo[key] = moment(photo[key]);
-      }
-    }
-    return getFacesForPhoto(photo.id).then((faces) => {
-      photo.faces = faces;
-      return res.status(200).json(photo);
-    })
   });
-})
+  
+  if (!results.length) {
+    return res.status(404).send({ message: id + " not found." });
+  }
+
+  if (id) {
+    if (results[0].duplicate) {
+      id = results[0].duplicate;
+    }
+  } else {
+    id = results[Math.floor(Math.random() * results.length)].id;
+  }
+
+  const photos = await photoDB.sequelize.query(
+    "SELECT photos.*,albums.path AS path FROM photos " +
+    "INNER JOIN albums ON albums.id=photos.albumId " +
+    "WHERE photos.duplicate=0 AND photos.deleted=0 AND photos.scanned NOT NULL AND photos.id=:id", {
+    replacements: { id },
+    type: photoDB.Sequelize.QueryTypes.SELECT,
+    raw: true
+  });
+
+  if (photos.length === 0) {
+    return res.status(404).send({ message: `Error obtaining ${id}.`});
+  }
+   
+  const photo = photos[0];
+  for (var key in photo) {
+    if (photo[key] instanceof Date) {
+      photo[key] = moment(photo[key]);
+    }
+  }
+  
+  const faces = await getFacesForPhoto(photo.id);
+  photo.faces = faces;
+  return res.status(200).json(photo);
+});
 
 router.get("/mvimg/*", function(req, res/*, next*/) {
   let limit = parseInt(req.query.limit) || 50,