.gitignore

@@ -1,3 +1 @@
 .env
-node_modules
-frontend/auto-clusters.html

Dockerfile

@@ -50,4 +50,5 @@ COPY /frontend /website/frontend
 COPY /db /website/db
 COPY /config /website/config
 
+
 CMD [ "/entrypoint.sh" ]

config/default.json

@@ -25,7 +25,7 @@
   },
   "picturesPath": "./pictures",
   "basePath": "/photos",
-  "facesPath": "./pictures/face-data",
+  "faceData": "./pictures/face-data",
   "sessions": {
     "db": "sessions.db",
     "store-secret": "234j23jffj23f!41$@#!1113j3"

docker-compose.yml

@@ -2,16 +2,16 @@ version: '3.1'
 
 services:
   photos:
-    env_file:
-      - .env
     build: .
     image: photos:latest
-    container_name: ${CONTAINER}photos
+    container_name: photos
  #   depends_on:
  #     - db
     restart: always
+    env_file:
+      - .env
     ports:
-      - ${PORT}:8123
+      - ${PORT}:${PORT}
     volumes:
       - ${PICTURES}:/pictures
       - ${PWD}/db:/db

ketrface/cluster.py

@@ -12,15 +12,9 @@ import functools
 from ketrface.util import *
 from ketrface.dbscan import *
 from ketrface.db import *
-from ketrface.config import *
 
-config = read_config()
-
-html_path = merge_config_path(config['path'], 'frontend')
-pictures_path = merge_config_path(config['path'], config['picturesPath'])
-faces_path = merge_config_path(config['path'], config['facesPath'])
-db_path = merge_config_path(config['path'], config["db"]["photos"]["host"])
-html_base = config['basePath']
+html_base = '../'
+db_path = '../db/photos.db'
 
 # TODO
 # Switch to using DBSCAN
@@ -52,7 +46,7 @@ def gen_html(identities):
         label = f'Cluster ({face["cluster"]["id"]})'
      
       print('<div style="position:relative;display:inline-flex;flex-direction:column">')
-      path = f'{html_base}/faces/{"{:02d}".format(faceId % 100)}'
+      path = f'{html_base}/faces/{"{:02d}".format(faceId % 10)}'
       print(f'<img src="{path}/{faceId}.jpg"/>')
       print(f'<div style="background-color:rgba(255, 255, 255, 0.4);position:absolute;top:0px;left:0px;right:0px;padding:0.25rem">{label}: {distance}</div>')
       print(f'<div style="background-color:rgba(255, 255, 255, 0.4);position:absolute;bottom:0px;left:0px;right:0px;padding:0.25rem">{faceId} {photoId} {confidence} {focus}</div>')
@@ -140,17 +134,7 @@ def cluster_sort(A, B):
   elif diff < 0:
     return -1
   return 0
-
-def build_straglers(faces):
-  noise = []
-  undefined = []
-  for face in faces:
-    if face['cluster'] == Noise:
-      noise.append(face)
-    elif face['cluster'] == Undefined:
-      undefined.append(face)
-  return noise + undefined
-
+  
 print('Loading faces from database')
 faces = load_faces()
 print(f'{len(faces)} faces loaded')
@@ -175,13 +159,13 @@ while removed != 0:
   if removed > 0:
     print(f'Excluded {removed} faces this epoch')
 
+
+
 print(f'{len(identities)} identities seeded.')
 
 # Cluster the clusters...
 print('Reducing clusters via DBSCAN')
 reduced = DBSCAN(identities, eps = MAX_CLUSTER_DISTANCE, minPts = 2)
-if len(reduced) == 0:
-  reduced = identities
 # For each cluster, merge the lists of faces referenced in the cluster's
 # "faces" field, which is pointing to clusters (and not actual faces)
 for cluster in reduced:
@@ -193,22 +177,19 @@ for cluster in reduced:
 # Creating a set containing those faces which have not been bound
 # to an identity to recluster them in isolation from the rest of
 # the faces
-straglers = build_straglers(faces)
-reduced = reduced + DBSCAN(straglers)
+noise = []
+undefined = []
+clustered = []
+for face in faces:
+  if face['cluster'] == Noise:
+    noise.append(face)
+  elif face['cluster'] == Undefined:
+    undefined.append(face)
 
-# Build a final cluster with all remaining uncategorized faces
-remaining_cluster = {
-  'id': len(reduced) + 1,
-  'distance': 0,
-  'descriptors': [],
-  'cluster': Undefined,
-  'faces': []
-}
-straglers = build_straglers(faces)
-for face in straglers:
-  face['cluster'] = remaining_cluster
-  remaining_cluster['faces'].append(face)
-reduced.append(remaining_cluster)
+print(f'Stats: Noise = {len(noise)}, Undefined = {len(undefined)}')
+
+straglers = DBSCAN(noise + undefined)
+reduced = update_cluster_averages(reduced + straglers)
 
 # Give all merged identity lists a unique ID
 for id, identity in enumerate(reduced):
@@ -216,8 +197,6 @@ for id, identity in enumerate(reduced):
   for face in identity['faces']:
     face['cluster'] = identity
 
-reduced = update_cluster_averages(reduced)
-
 update_distances(reduced)
 
 sort_identities(reduced)
@@ -237,7 +216,7 @@ for i, A in enumerate(reduced):
       distance = "{:0.4f}".format(distance)
       print(f'{A["id"]} to {B["id"]} = {distance}: MERGE')
 
-print('Writing to "auto-clusters.html"')
-redirect_on(os.path.join(html_path, 'auto-clusters.html'))
+print('Writing to "identities.html"')
+redirect_on('identities.html')
 gen_html(reduced)
 redirect_off()

ketrface/detect.py

@@ -11,16 +11,8 @@ import numpy as np
 import cv2
 from ketrface.util import *
 from ketrface.db import *
-from ketrface.config import *
-
-config = read_config()
-
-html_path = merge_config_path(config['path'], 'frontend')
-pictures_path = merge_config_path(config['path'], config['picturesPath'])
-faces_path = merge_config_path(config['path'], config['facesPath'])
-db_path = merge_config_path(config['path'], config["db"]["photos"]["host"])
-html_base = config['basePath']
 
+face_base = '../'
 model_name = 'VGG-Face'    # 'ArcFace'
 detector_backend = 'mtcnn' # 'retinaface'
 model = DeepFace.build_model(model_name)
@@ -245,9 +237,9 @@ with conn:
         'descriptorId': faceDescriptorId,
       })
 
-      path = f'{faces_path}/{"{:02d}".format(faceId % 100)}'
+      path = f'{face_base}faces/{"{:02d}".format(faceId % 10)}'
       try:
-        os.makedirs(path)
+        os.mkdir(path)
       except FileExistsError:
         pass
         
@@ -260,9 +252,6 @@ with conn:
       exif_ifd = {piexif.ExifIFD.UserComment: compressed_str}
       exif_dict = {"0th": {}, "Exif": exif_ifd, "1st": {},
         "thumbnail": None, "GPS": {}}
-      image.save(
-        f'{path}/{faceId}.jpg', 
-        quality = 'maximum', 
-        exif = piexif.dump(exif_dict))
+      image.save(f'{path}/{faceId}.jpg', exif = piexif.dump(exif_dict))
 
     update_face_count(conn, photoId, len(faces))

ketrface/ketrface/config.py

@@ -1,64 +0,0 @@
-import os
-import json
-import re
-
-def dict_merge(dct, merge_dct):
-  """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
-  updating only top-level keys, dict_merge recurses down into dicts nested
-  to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
-  ``dct``.
-  :param dct: dict onto which the merge is executed
-  :param merge_dct: dct merged into dct
-  :return: None
-  """
-  for k, v in merge_dct.items():
-    if (k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], dict)):  #noqa
-      dict_merge(dct[k], merge_dct[k])
-    else:
-      dct[k] = merge_dct[k]
-  return dct
-
-def read_config():
-  path = os.path.normpath(os.getcwd())
-  res = {}
-  file = None
-  while file == None:
-    try:
-      config_path = os.path.join(path, 'config', 'default.json')
-      file = open(config_path, 'r')
-      break
-    except:
-      before = path
-      path = os.path.normpath(os.path.join(path, '..'))
-      if before == path:
-        break
-
-  if file is None:
-    return res
-
-  res['path'] = path
-
-  data = json.load(file)
-  file.close()
-  dict_merge(res, data)
-
-  try:
-    config_path = os.path.join(path, 'config', 'local.json')
-    file = open(config_path, 'r')
-  except:
-    file = None
-
-  if file is None:
-    return res
-
-  data = json.load(file)
-  file.close()
-  dict_merge(res, data)
-
-  return res
-
-def merge_config_path(config_path, path):
-  if path[0] == '/':
-    return os.path.normpath(path)
-  return os.path.normpath(os.path.join(config_path, path))
-  

server/app.js

@@ -16,7 +16,6 @@ const express = require("express"),
 require("./console-line.js"); /* Monkey-patch console.log with line numbers */
 
 const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
-  facesPath = config.get("facesPath").replace(/\/$/, "") + "/",
   serverConfig = config.get("server");
 
 let basePath = config.get("basePath");
@@ -26,9 +25,8 @@ if (basePath == "//") {
 }
 
 let photoDB = null, userDB = null;
-console.log(`Loading pictures out of: ${picturesPath}`);
-console.log(`Loading faces out of: ${facesPath} (mapped to ${basePath}faces})`);
-console.log(`Hosting server from: ${basePath}`);
+console.log("Loading pictures out of: " + picturesPath);
+console.log("Hosting server from: " + basePath);
 
 const app = express();
 
@@ -240,7 +238,7 @@ app.use(basePath, index);
 const users = require("./routes/users");
 app.use(basePath + "api/v1/users", users.router);
 
-app.use((err, req, res, next) => {
+app.use(function(err, req, res, next) {
   res.status(err.status || 500).json({
     message: err.message,
     error: {}
@@ -248,7 +246,7 @@ app.use((err, req, res, next) => {
 });
 
 /* Check authentication */
-app.use(basePath, (req, res, next) => {
+app.use(basePath, function(req, res, next) {
   return users.getSessionUser(req).then(function(user) {
     if (user.restriction) {
       return res.status(401).send(user.restriction);
@@ -260,11 +258,6 @@ app.use(basePath, (req, res, next) => {
   });
 });
 
-app.use(`${basePath}faces/`, express.static(facesPath, {
-  maxAge: '14d',
-  index: false
-}));
-
 /* Everything below here requires a successful authentication */
 app.use(basePath, express.static(picturesPath, { 
   maxAge: '14d',

server/face-recognizer.js

@@ -37,7 +37,7 @@ const maxConcurrency = require("os").cpus().length;
 require("./console-line.js"); /* Monkey-patch console.log with line numbers */
 
 const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
-  facesPath = config.get("facesPath").replace(/\/$/, "") + "/";
+  faceData = picturesPath + "face-data/";
 
 let photoDB = null;
 
@@ -137,7 +137,7 @@ require("./db/photos").then(function(db) {
       return Promise.map(faces, (face) => {
         return Promise.map([ "-data.json", "-original.png" ], (suffix) => {
           const id = face.id,
-            dataPath = facesPath + (id % 100) + "/" + id + suffix;
+            dataPath = faceData + (id % 100) + "/" + id + suffix;
           return exists(dataPath).then((result) => {
             if (result) {
               console.log(`...removing ${dataPath}`);
@@ -185,7 +185,7 @@ require("./db/photos").then(function(db) {
         }).spread((results, metadata) => {
           return metadata.lastID;
         }).then((id) => {
-          const path = facesPath + (id % 100);
+          const path = faceData + (id % 100);
           return mkdir(path).then(() => {
             const dataPath = `${path}/${id}-data.json`, data = [];
             console.log(`...writing descriptor data to ${dataPath}...`);
@@ -280,7 +280,7 @@ require("./db/photos").then(function(db) {
       console.log(`...reading ${allFaces.length} descriptors...`);
       return Promise.map(allFaces, (face) => {
         const id = face.id,
-          dataPath = facesPath + "/" + (id % 100) + "/" + id + "-data.json";
+          dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
 
         if (id in descriptors) {
           return;

server/face.js

@@ -22,7 +22,7 @@ const maxConcurrency = require("os").cpus().length;
 require("./console-line.js"); /* Monkey-patch console.log with line numbers */
 
 const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
-  facesPath = config.get("facesPath").replace(/\/$/, "") + "/";
+  faceData = picturesPath + "face-data/";
 
 function alignFromLandmarks(image, landmarks, drawLandmarks) {
   const faceMargin = 0.45,
@@ -151,7 +151,7 @@ require("./db/photos").then(function(db) {
                 _height: (photo.bottom - photo.top) * photo.height,
               }
             },
-            descriptor: JSON.parse(fs.readFileSync(facesPath + (id % 100) + "/" + id + "-data.json"))
+            descriptor: JSON.parse(fs.readFileSync(faceData + (id % 100) + "/" + id + "-data.json"))
           } ];
           return [ file, image, detectors ];
         });