Adding face-recognition backend

Signed-off-by: James Ketrenos <james_gitlab@ketrenos.com>

.dockerignore

@@ -1,2 +1,17 @@
 *
+!config
+!db
+!docker-compose.yml
+!Dockerfile
 !entrypoint.sh
+!face.js
+!frontend
+!models
+!package.json
+!package-lock.json
+!password.js
+!query.sh
+!README.md
+!reset-db.sh
+!server
+!util

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:disco
+FROM ubuntu:eoan
 
 RUN apt-get update 
 
@@ -17,7 +17,10 @@ RUN DEBIAN_FRONTEND=NONINTERACTIVE apt-get install -y \
 RUN npm install --global npm@latest npx
 
 # Speed up face-recognition and dev tools
-RUN apt-get install -y libopenblas-dev cmake
+RUN apt-get install -y libopenblas-dev
+
+# Required for dlib to build
+RUN apt-get install -y libx11-dev libpng16-16
 
 # NEF processing uses ufraw-batch
 RUN apt-get install -y ufraw-batch
@@ -40,17 +43,16 @@ RUN groupadd -g 1000 user \
 # Set 'sudo' to NOPASSWD for all container users
 RUN sed -i -e 's,%sudo.*,%sudo ALL=(ALL) NOPASSWD:ALL,g' /etc/sudoers
 
-COPY /entrypoint.sh /entrypoint.sh
-
-RUN DEBIAN_FRONTEND=noninteractive \
-  && apt-get install --no-install-recommends -y \
-    git
-
 RUN DEBIAN_FRONTEND=noninteractive \
+  && apt-get update \
   && apt-get install --no-install-recommends -y \
+    git \
     sqlite3
 
-USER user
-WORKDIR /website
+COPY . /website
 
-CMD [ "/entrypoint.sh" ]
+#USER user
+WORKDIR /website
+RUN npm install
+
+CMD [ "/website/entrypoint.sh" ]

config/default.json

@@ -4,14 +4,16 @@
       "host": "sqlite:db/photos.db",
       "options": {
         "logging" : false,
-        "timezone": "+00:00"
+        "timezone": "+00:00",
+        "operatorsAliases": false
       }
     },
     "users": {
       "host": "sqlite:db/users.db",
       "options": {
         "logging" : false,
-        "timezone": "+00:00"
+        "timezone": "+00:00",
+        "operatorsAliases": false
       }
     }
   },

package.json

@@ -4,7 +4,8 @@
   "description": "Self hosting photo",
   "main": "server/app.js",
   "scripts": {
-    "start": "node ./server/app.js"
+    "start": "node ./server/app.js",
+    "faces": "node ./server/face-recognizer.js"
   },
   "repository": {
     "type": "git",
@@ -16,28 +17,28 @@
     "face-recognition": "^0.9.4"
   },
   "dependencies": {
-    "bluebird": "^3.5.3",
-    "body-parser": "^1.18.3",
+    "bluebird": "^3.7.2",
+    "body-parser": "^1.19.0",
     "config": "^1.31.0",
     "connect-sqlite3": "^0.9.11",
-    "cookie-parser": "^1.4.3",
+    "cookie-parser": "^1.4.4",
     "exif-reader": "github:paras20xx/exif-reader",
-    "express": "^4.16.4",
-    "express-session": "^1.15.6",
-    "handlebars": "^4.0.12",
-    "ldapauth-fork": "^4.0.2",
+    "express": "^4.17.1",
+    "express-session": "^1.17.0",
+    "handlebars": "^4.5.3",
+    "ldapauth-fork": "^4.2.0",
     "ldapjs": "^1.0.2",
     "mariasql": "^0.2.6",
-    "moment": "^2.22.2",
+    "moment": "^2.24.0",
     "moment-holiday": "^1.5.1",
     "morgan": "^1.9.1",
-    "mustache": "^3.0.1",
+    "mustache": "^3.2.1",
     "nodemailer": "^4.7.0",
-    "qs": "^6.6.0",
-    "sequelize": "^4.41.2",
+    "qs": "^6.9.1",
+    "sequelize": "^4.44.3",
     "sequelize-mysql": "^1.7.0",
     "sharp": "^0.20.8",
-    "sqlite3": "^4.0.4"
+    "sqlite3": "^4.1.1"
   },
   "jshintConfig": {
     "undef": true,

server/app.js

@@ -2,10 +2,6 @@
 
 process.env.TZ = "Etc/GMT";
 
-if (process.env.LOG_LINE) {
-  require("./monkey.js"); /* monkey patch console.log */
-}
-
 console.log("Loading photos.ketr");
 
 const express = require("express"),
@@ -95,7 +91,9 @@ app.use(function(req, res, next){
 app.use(session({
   store: new SQLiteStore({ db: config.get("sessions.db") }),
   secret: config.get("sessions.store-secret"),
-  cookie: { maxAge: 7 * 24 * 60 * 60 * 1000 } // 1 week
+  cookie: { maxAge: 7 * 24 * 60 * 60 * 1000 }, // 1 week
+  saveUninitialized: false,
+  resave: true
 }));
 
 const index = require("./routes/index");

server/db/photos.js

@@ -62,6 +62,10 @@ function init() {
       width: Sequelize.INTEGER,
       height: Sequelize.INTEGER,
       size: Sequelize.INTEGER,
+      faces: {
+        type: Sequelize.INTEGER,
+        defaultValue: -1 /* not scanned */
+      },
       duplicate: {
         type: Sequelize.BOOLEAN,
         defaultValue: 0
@@ -82,6 +86,61 @@ function init() {
       timestamps: false
     });
 
+    const Identity = db.sequelize.define('identity', {
+      id: {
+        type: Sequelize.INTEGER,
+        primaryKey: true,
+        autoIncrement: true
+      },
+      lastName: Sequelize.STRING,
+      firstName: Sequelize.STRING,
+      middleName: Sequelize.STRING,
+      name: {
+        type: Sequelize.STRING,
+        allowNull: false
+      }
+    }, {
+      timestamps: false
+    });
+
+    const Face = db.sequelize.define('face', {
+      id: {
+        type: Sequelize.INTEGER,
+        primaryKey: true,
+        autoIncrement: true
+      },
+      photoId: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        references: {
+          model: Photo,
+          key: 'id',
+        }
+      },
+      identityId: {
+        type: Sequelize.INTEGER,
+        allowNull: true,
+        references: {
+          model: Identity,
+          key: 'id',
+        }
+      },
+      identityDistance: { /* How far are markers from identity match? */
+        type: Sequelize.DOUBLE,
+        defaultValue: -1.0
+      },
+      faceConfidence: { /* How confident that this is a face? */
+        type: Sequelize.DOUBLE,
+        defaultValue: 0
+      },
+      top: Sequelize.FLOAT, /* 0..1 * photo.height */
+      left: Sequelize.FLOAT, /* 0..1 * photo.width */
+      bottom: Sequelize.FLOAT, /* 0..1 * photo.height */
+      right: Sequelize.FLOAT, /* 0..1 * photo.width */
+    }, {
+      timestamps: false
+    });
+
     const PhotoHash = db.sequelize.define('photohash', {
       hash: {
         type: Sequelize.STRING,

server/face-recognizer.js

@@ -0,0 +1,127 @@
+/*
+* Face recognition:
+* 1. For each photo, extract all faces. Store face rectangles.
+*    face_id unique
+*    photo_id foreign key
+*    top left bottom right
+*    identity_id
+*    distance (0 == truth; manually assigned identity)
+* 2. For each face_id, create:
+*    /${picturesPath}face-data/${face_id % 100}/
+*      ${face_id}-normalized
+*      ${face_id}-original
+*      ${face_id}-data
+*/
+
+"use strict";
+
+process.env.TZ = "Etc/GMT";
+
+console.log("Loading face-recognizer");
+
+const config = require("config"),
+  Promise = require("bluebird"),
+  { mkdir, unlink } = require("./lib/util"),
+  fr = require("face-recognition");
+
+require("./console-line.js"); /* Monkey-patch console.log with line numbers */
+
+const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
+  faceData = picturesPath + "face-data/";
+
+let photoDB = null, faceDataExists = false;
+
+console.log("Loading pictures out of: " + picturesPath);
+
+require("./db/photos").then(function(db) {
+  photoDB = db;
+}).then(() => {
+  console.log("DB connected.");
+}).then(() => {
+  console.log("Beginning face detection scanning.");
+  return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " +
+    "FROM photos " +
+    "LEFT JOIN albums ON (albums.id=photos.albumId) " +
+    "WHERE faces=-1 ORDER BY albums.path,photos.filename", {
+      type: photoDB.sequelize.QueryTypes.SELECT
+    }
+  ).then((results) => {
+    console.log(`${results.length} photos have not had faces scanned.`);
+    return Promise.map(results, (photo) => {
+      const filePath = photo.path + photo.filename;
+      console.log(`Processing ${filePath}...`);
+
+      return photoDB.sequelize.transaction(function(transaction) {
+        /* Remove any existing face data for this photo */
+        return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
+          transaction: transaction,
+          replacements: photo,
+        }).then((faces) => {
+          /* For each face-id, remove any face-data files, and then remove all the entries
+           * from the DB */
+          return Promise.map(faces, (id) => {
+            return Promise.mapSeries(["-normalized.png", "-data.json" ], (fileSuffix) => {
+              const filePath = faceData + "/" + (id % 100) + "/" + id + fileSuffix;
+              return exists(filePath).then((result) => {
+                console.log(`...removing ${filePath}`);
+                return unlink(filePath);
+              });
+            });
+          }).then(() => {
+            return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
+              transaction: transaction,
+              replacements: photo,
+            });
+          }).then(() => {
+            const image = fr.loadImage(filePath),
+              detector = fr.FaceDetector();
+            
+            console.log("...detecting faces.");
+            const faceRectangles = detector.locateFaces(image)
+            if (faceRectangles.length == 0) {
+              console.log("...no faces found in image.");
+              return;
+            }
+          
+            /* Create a face entry in photos for each face found. */
+            const faceImages = detector.detectFaces(image, 200)
+            
+            console.log(`...saving ${faceImages.length} faces.`);
+
+            return Promise.map(faceRectangles, (face, index) => {
+              return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
+                "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
+                replacements: {
+                  id: photo.id,
+                  top: face.top / photo.height,
+                  left: face.left / photo.width,
+                  bottom: face.top / photo.height,
+                  right: face.right / photo.width,
+                  faceConfidence: face.confidence
+                },
+                transaction: transaction
+              }).spread((results, metadata) => {
+                return metadata.lastID;
+              }).then((id) => {
+                console.log(`...DB id ${id}. Writing data and images...`);
+                const filePathPrefix = faceData + "/" + (id % 100) + "/" + id;
+                /* https://medium.com/@ageitgey/machine-learning-is-fun-part-4-modern-face-recognition-with-deep-learning-c3cffc121d78 */
+                const data = [];
+                for (let i = 0; i < 128; i++) {
+                  data.push(Math.random() - 0.5);
+                }
+                fs.writeFileSync(filePathPrefix + "-data.json", JSON.stringify(data));
+                fr.saveImage(filePathPrefix + "-normalized.png", faceImages[index]);
+              });
+            });
+          });
+        });
+      });
+    });
+  });
+}).then(() => {
+  console.log("Face detection scanning completed.");
+}).catch((error) => {
+  console.error(error);
+  process.exit(-1);
+});

server/lib/util.js

@@ -0,0 +1,88 @@
+"use strict";
+
+const config = require("config"),
+  fs = require("fs"),
+  Promise = require("bluebird"),
+  picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
+
+const stat = function (_path) {
+  if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) {
+    _path = _path.substring(picturesPath.length);
+  }
+
+  let path = picturesPath + _path;
+
+  return new Promise(function (resolve, reject) {
+    fs.stat(path, function (error, stats) {
+      if (error) {
+        return reject(error);
+      }
+      return resolve(stats);
+    });
+  });
+}
+
+const unlink = function (_path) {
+  if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) {
+    _path = _path.substring(picturesPath.length);
+  }
+
+  let path = picturesPath + _path;
+
+  return new Promise(function (resolve, reject) {
+    fs.unlink(path, function (error) {
+      if (error) {
+        return reject(error);
+      }
+      return resolve();
+    });
+  });
+}
+
+const mkdir = function (_path) {
+  if (_path.indexOf(picturesPath) == 0) {
+    _path = _path.substring(picturesPath.length);
+  }
+
+  let parts = _path.split("/"), path;
+  
+  parts.unshift(picturesPath);
+  return Promise.mapSeries(parts, function (part) {
+    if (!path) {
+      path = picturesPath.replace(/\/$/, "");
+    } else {
+      path += "/" + part;
+    }
+
+    return stat(path).catch(function (error) {
+      if (error.code != "ENOENT") {
+        throw error;
+      }
+
+      return new Promise(function (resolve, reject) {
+        fs.mkdir(path, function (error) {
+          if (error) {
+            return reject(error);
+          }
+
+          return resolve();
+        });
+      });
+    });
+  });
+}
+
+const exists = function(path) {
+  return stat(path).then(function() {
+    return true;
+  }).catch(function() {
+    return false;
+  });
+}
+
+module.exports = {
+  stat,
+  exists,
+  mkdir,
+  unlink
+};

server/monkey.js

@@ -1,29 +0,0 @@
-/* monkey-patch console.log to prefix with file/line-number */
-function lineLogger(logFn) {
-  let cwd = process.cwd(),
-    cwdRe = new RegExp("^[^/]*" + cwd.replace("/", "\\/") + "\/([^:]*:[0-9]*).*$");
-
-  function getErrorObject() {
-    try {
-      throw Error();
-    } catch (err) {
-      return err;
-    }
-  }
-
-  let err = getErrorObject(),
-    caller_line = err.stack.split("\n")[4],
-    args = [caller_line.replace(cwdRe, "$1 -")];
-
-  /* arguments.unshift() doesn't exist... */
-  for (var i = 1; i < arguments.length; i++) {
-    args.push(arguments[i]);
-  }
-
-  logFn.apply(this, args);
-}
-
-console.log = lineLogger.bind(console, console.log);
-console.warn = lineLogger.bind(console, console.warn);
-console.error = lineLogger.bind(console, console.error);
-

server/scanner.js

@@ -1,11 +1,29 @@
+/**
+ * scanner
+ * 
+ * Face recognition:
+ * 1. For each photo, extract all faces. Store face rectangles.
+ *    face_id unique
+ *    photo_id foreign key
+ *    top left bottom right
+ *    identity_id
+ *    distance (0 == truth; manually assigned identity)
+ * 2. For each face_id, create:
+ *    normalized_file
+ *    original_file
+ *    128 float 
+ */
 "use strict";
 
+/* meta directories are not scanned for photos */
+const metaDirectories = [ "thumbs", "raw", "face-data", ".git", "corrupt" ];
+
 const Promise = require("bluebird"),
   fs = require("fs"),
   config = require("config"),
   moment = require("moment"),
-  crypto = require("crypto");
-
+  crypto = require("crypto"),
+  { stat, mkdir, exists } = require("./lib/util");
 
 let photoDB = null;
 
@@ -42,63 +60,6 @@ let processRunning = false;
 const { spawn } = require('child_process');
 
 const sharp = require("sharp"), exif = require("exif-reader");
-const stat = function (_path) {
-  if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) {
-    _path = _path.substring(picturesPath.length);
-  }
-
-  let path = picturesPath + _path;
-
-  return new Promise(function (resolve, reject) {
-    fs.stat(path, function (error, stats) {
-      if (error) {
-        return reject(error);
-      }
-      return resolve(stats);
-    });
-  });
-}
-
-const mkdir = function (_path) {
-  if (_path.indexOf(picturesPath) == 0) {
-    _path = _path.substring(picturesPath.length);
-  }
-
-  let parts = _path.split("/"), path;
-  
-  parts.unshift(picturesPath);
-  return Promise.mapSeries(parts, function (part) {
-    if (!path) {
-      path = picturesPath.replace(/\/$/, "");
-    } else {
-      path += "/" + part;
-    }
-
-    return stat(path).catch(function (error) {
-      if (error.code != "ENOENT") {
-        throw error;
-      }
-
-      return new Promise(function (resolve, reject) {
-        fs.mkdir(path, function (error) {
-          if (error) {
-            return reject(error);
-          }
-
-          return resolve();
-        });
-      });
-    });
-  });
-}
-
-const exists = function(path) {
-  return stat(path).then(function() {
-    return true;
-  }).catch(function() {
-    return false;
-  });
-}
 
 function convertRawToJpg(path, file) {
   setStatus("Converting " + path + file);
@@ -464,8 +425,8 @@ function scanDir(parent, path) {
         return resolve([]);
       }
 
-      /* Remove 'thumbs' and 'raw' directories from being processed */
-      files = files.filter(function(file) {
+      /* Remove meta-data directories from being processed */
+      files = files.filter((file) => {
         for (var i = 0; i < files.length; i++) {
           /* If this file has an original NEF/ORF on the system, don't add the JPG to the DB */
           if (rawExtension.exec(files[i]) && file == files[i].replace(rawExtension, ".jpg")) {
@@ -480,8 +441,7 @@ function scanDir(parent, path) {
             return false;
           }
         }
-
-        return file != "raw" && file != "thumbs" && file != ".git" && file != "corrupt";
+        return metaDirectories.indexOf(file) == -1;
       });
 
       return resolve(files);