From 6f51d5dc4d6a378f600111806bf9313e868d49ca Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Fri, 3 Jan 2020 15:30:50 -0800 Subject: [PATCH] Adding face-recognition backend Signed-off-by: James Ketrenos --- .dockerignore | 15 +++++ Dockerfile | 24 +++---- config/default.json | 6 +- package.json | 27 ++++---- server/app.js | 8 +-- server/db/photos.js | 59 ++++++++++++++++++ server/face-recognizer.js | 127 ++++++++++++++++++++++++++++++++++++++ server/lib/util.js | 88 ++++++++++++++++++++++++++ server/monkey.js | 29 --------- server/scanner.js | 86 +++++++------------------- 10 files changed, 346 insertions(+), 123 deletions(-) create mode 100644 server/face-recognizer.js create mode 100644 server/lib/util.js delete mode 100644 server/monkey.js diff --git a/.dockerignore b/.dockerignore index 864179f..3aca41b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,17 @@ * +!config +!db +!docker-compose.yml +!Dockerfile !entrypoint.sh +!face.js +!frontend +!models +!package.json +!package-lock.json +!password.js +!query.sh +!README.md +!reset-db.sh +!server +!util diff --git a/Dockerfile b/Dockerfile index 9b468f3..44039c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:disco +FROM ubuntu:eoan RUN apt-get update @@ -17,7 +17,10 @@ RUN DEBIAN_FRONTEND=NONINTERACTIVE apt-get install -y \ RUN npm install --global npm@latest npx # Speed up face-recognition and dev tools -RUN apt-get install -y libopenblas-dev cmake +RUN apt-get install -y libopenblas-dev + +# Required for dlib to build +RUN apt-get install -y libx11-dev libpng16-16 # NEF processing uses ufraw-batch RUN apt-get install -y ufraw-batch @@ -40,17 +43,16 @@ RUN groupadd -g 1000 user \ # Set 'sudo' to NOPASSWD for all container users RUN sed -i -e 's,%sudo.*,%sudo ALL=(ALL) NOPASSWD:ALL,g' /etc/sudoers -COPY /entrypoint.sh /entrypoint.sh - -RUN DEBIAN_FRONTEND=noninteractive \ - && apt-get install --no-install-recommends -y \ - git - RUN DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ && apt-get install --no-install-recommends -y \ + git \ sqlite3 -USER user -WORKDIR /website +COPY . /website -CMD [ "/entrypoint.sh" ] +#USER user +WORKDIR /website +RUN npm install + +CMD [ "/website/entrypoint.sh" ] diff --git a/config/default.json b/config/default.json index 393573b..13fc7a1 100644 --- a/config/default.json +++ b/config/default.json @@ -4,14 +4,16 @@ "host": "sqlite:db/photos.db", "options": { "logging" : false, - "timezone": "+00:00" + "timezone": "+00:00", + "operatorsAliases": false } }, "users": { "host": "sqlite:db/users.db", "options": { "logging" : false, - "timezone": "+00:00" + "timezone": "+00:00", + "operatorsAliases": false } } }, diff --git a/package.json b/package.json index 5f3d948..e40041a 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "Self hosting photo", "main": "server/app.js", "scripts": { - "start": "node ./server/app.js" + "start": "node ./server/app.js", + "faces": "node ./server/face-recognizer.js" }, "repository": { "type": "git", @@ -16,28 +17,28 @@ "face-recognition": "^0.9.4" }, "dependencies": { - "bluebird": "^3.5.3", - "body-parser": "^1.18.3", + "bluebird": "^3.7.2", + "body-parser": "^1.19.0", "config": "^1.31.0", "connect-sqlite3": "^0.9.11", - "cookie-parser": "^1.4.3", + "cookie-parser": "^1.4.4", "exif-reader": "github:paras20xx/exif-reader", - "express": "^4.16.4", - "express-session": "^1.15.6", - "handlebars": "^4.0.12", - "ldapauth-fork": "^4.0.2", + "express": "^4.17.1", + "express-session": "^1.17.0", + "handlebars": "^4.5.3", + "ldapauth-fork": "^4.2.0", "ldapjs": "^1.0.2", "mariasql": "^0.2.6", - "moment": "^2.22.2", + "moment": "^2.24.0", "moment-holiday": "^1.5.1", "morgan": "^1.9.1", - "mustache": "^3.0.1", + "mustache": "^3.2.1", "nodemailer": "^4.7.0", - "qs": "^6.6.0", - "sequelize": "^4.41.2", + "qs": "^6.9.1", + "sequelize": "^4.44.3", "sequelize-mysql": "^1.7.0", "sharp": "^0.20.8", - "sqlite3": "^4.0.4" + "sqlite3": "^4.1.1" }, "jshintConfig": { "undef": true, diff --git a/server/app.js b/server/app.js index a3d78ce..7888fa3 100755 --- a/server/app.js +++ b/server/app.js @@ -2,10 +2,6 @@ process.env.TZ = "Etc/GMT"; -if (process.env.LOG_LINE) { - require("./monkey.js"); /* monkey patch console.log */ -} - console.log("Loading photos.ketr"); const express = require("express"), @@ -95,7 +91,9 @@ app.use(function(req, res, next){ app.use(session({ store: new SQLiteStore({ db: config.get("sessions.db") }), secret: config.get("sessions.store-secret"), - cookie: { maxAge: 7 * 24 * 60 * 60 * 1000 } // 1 week + cookie: { maxAge: 7 * 24 * 60 * 60 * 1000 }, // 1 week + saveUninitialized: false, + resave: true })); const index = require("./routes/index"); diff --git a/server/db/photos.js b/server/db/photos.js index aa717e6..5d9f6d7 100755 --- a/server/db/photos.js +++ b/server/db/photos.js @@ -62,6 +62,10 @@ function init() { width: Sequelize.INTEGER, height: Sequelize.INTEGER, size: Sequelize.INTEGER, + faces: { + type: Sequelize.INTEGER, + defaultValue: -1 /* not scanned */ + }, duplicate: { type: Sequelize.BOOLEAN, defaultValue: 0 @@ -82,6 +86,61 @@ function init() { timestamps: false }); + const Identity = db.sequelize.define('identity', { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true + }, + lastName: Sequelize.STRING, + firstName: Sequelize.STRING, + middleName: Sequelize.STRING, + name: { + type: Sequelize.STRING, + allowNull: false + } + }, { + timestamps: false + }); + + const Face = db.sequelize.define('face', { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true + }, + photoId: { + type: Sequelize.INTEGER, + allowNull: false, + references: { + model: Photo, + key: 'id', + } + }, + identityId: { + type: Sequelize.INTEGER, + allowNull: true, + references: { + model: Identity, + key: 'id', + } + }, + identityDistance: { /* How far are markers from identity match? */ + type: Sequelize.DOUBLE, + defaultValue: -1.0 + }, + faceConfidence: { /* How confident that this is a face? */ + type: Sequelize.DOUBLE, + defaultValue: 0 + }, + top: Sequelize.FLOAT, /* 0..1 * photo.height */ + left: Sequelize.FLOAT, /* 0..1 * photo.width */ + bottom: Sequelize.FLOAT, /* 0..1 * photo.height */ + right: Sequelize.FLOAT, /* 0..1 * photo.width */ + }, { + timestamps: false + }); + const PhotoHash = db.sequelize.define('photohash', { hash: { type: Sequelize.STRING, diff --git a/server/face-recognizer.js b/server/face-recognizer.js new file mode 100644 index 0000000..6035eab --- /dev/null +++ b/server/face-recognizer.js @@ -0,0 +1,127 @@ +/* +* Face recognition: +* 1. For each photo, extract all faces. Store face rectangles. +* face_id unique +* photo_id foreign key +* top left bottom right +* identity_id +* distance (0 == truth; manually assigned identity) +* 2. For each face_id, create: +* /${picturesPath}face-data/${face_id % 100}/ +* ${face_id}-normalized +* ${face_id}-original +* ${face_id}-data +*/ + +"use strict"; + +process.env.TZ = "Etc/GMT"; + +console.log("Loading face-recognizer"); + +const config = require("config"), + Promise = require("bluebird"), + { mkdir, unlink } = require("./lib/util"), + fr = require("face-recognition"); + +require("./console-line.js"); /* Monkey-patch console.log with line numbers */ + +const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/", + faceData = picturesPath + "face-data/"; + +let photoDB = null, faceDataExists = false; + +console.log("Loading pictures out of: " + picturesPath); + +require("./db/photos").then(function(db) { + photoDB = db; +}).then(() => { + console.log("DB connected."); +}).then(() => { + console.log("Beginning face detection scanning."); + return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " + + "FROM photos " + + "LEFT JOIN albums ON (albums.id=photos.albumId) " + + "WHERE faces=-1 ORDER BY albums.path,photos.filename", { + type: photoDB.sequelize.QueryTypes.SELECT + } + ).then((results) => { + console.log(`${results.length} photos have not had faces scanned.`); + return Promise.map(results, (photo) => { + const filePath = photo.path + photo.filename; + console.log(`Processing ${filePath}...`); + + return photoDB.sequelize.transaction(function(transaction) { + /* Remove any existing face data for this photo */ + return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", { + transaction: transaction, + replacements: photo, + }).then((faces) => { + /* For each face-id, remove any face-data files, and then remove all the entries + * from the DB */ + return Promise.map(faces, (id) => { + return Promise.mapSeries(["-normalized.png", "-data.json" ], (fileSuffix) => { + const filePath = faceData + "/" + (id % 100) + "/" + id + fileSuffix; + return exists(filePath).then((result) => { + console.log(`...removing ${filePath}`); + return unlink(filePath); + }); + }); + }).then(() => { + return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", { + transaction: transaction, + replacements: photo, + }); + }).then(() => { + const image = fr.loadImage(filePath), + detector = fr.FaceDetector(); + + console.log("...detecting faces."); + const faceRectangles = detector.locateFaces(image) + if (faceRectangles.length == 0) { + console.log("...no faces found in image."); + return; + } + + /* Create a face entry in photos for each face found. */ + const faceImages = detector.detectFaces(image, 200) + + console.log(`...saving ${faceImages.length} faces.`); + + return Promise.map(faceRectangles, (face, index) => { + return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " + + "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", { + replacements: { + id: photo.id, + top: face.top / photo.height, + left: face.left / photo.width, + bottom: face.top / photo.height, + right: face.right / photo.width, + faceConfidence: face.confidence + }, + transaction: transaction + }).spread((results, metadata) => { + return metadata.lastID; + }).then((id) => { + console.log(`...DB id ${id}. Writing data and images...`); + const filePathPrefix = faceData + "/" + (id % 100) + "/" + id; + /* https://medium.com/@ageitgey/machine-learning-is-fun-part-4-modern-face-recognition-with-deep-learning-c3cffc121d78 */ + const data = []; + for (let i = 0; i < 128; i++) { + data.push(Math.random() - 0.5); + } + fs.writeFileSync(filePathPrefix + "-data.json", JSON.stringify(data)); + fr.saveImage(filePathPrefix + "-normalized.png", faceImages[index]); + }); + }); + }); + }); + }); + }); + }); +}).then(() => { + console.log("Face detection scanning completed."); +}).catch((error) => { + console.error(error); + process.exit(-1); +}); diff --git a/server/lib/util.js b/server/lib/util.js new file mode 100644 index 0000000..dcbb545 --- /dev/null +++ b/server/lib/util.js @@ -0,0 +1,88 @@ +"use strict"; + +const config = require("config"), + fs = require("fs"), + Promise = require("bluebird"), + picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/"; + +const stat = function (_path) { + if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) { + _path = _path.substring(picturesPath.length); + } + + let path = picturesPath + _path; + + return new Promise(function (resolve, reject) { + fs.stat(path, function (error, stats) { + if (error) { + return reject(error); + } + return resolve(stats); + }); + }); +} + +const unlink = function (_path) { + if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) { + _path = _path.substring(picturesPath.length); + } + + let path = picturesPath + _path; + + return new Promise(function (resolve, reject) { + fs.unlink(path, function (error) { + if (error) { + return reject(error); + } + return resolve(); + }); + }); +} + +const mkdir = function (_path) { + if (_path.indexOf(picturesPath) == 0) { + _path = _path.substring(picturesPath.length); + } + + let parts = _path.split("/"), path; + + parts.unshift(picturesPath); + return Promise.mapSeries(parts, function (part) { + if (!path) { + path = picturesPath.replace(/\/$/, ""); + } else { + path += "/" + part; + } + + return stat(path).catch(function (error) { + if (error.code != "ENOENT") { + throw error; + } + + return new Promise(function (resolve, reject) { + fs.mkdir(path, function (error) { + if (error) { + return reject(error); + } + + return resolve(); + }); + }); + }); + }); +} + +const exists = function(path) { + return stat(path).then(function() { + return true; + }).catch(function() { + return false; + }); +} + +module.exports = { + stat, + exists, + mkdir, + unlink +}; \ No newline at end of file diff --git a/server/monkey.js b/server/monkey.js deleted file mode 100644 index 1c8d322..0000000 --- a/server/monkey.js +++ /dev/null @@ -1,29 +0,0 @@ -/* monkey-patch console.log to prefix with file/line-number */ -function lineLogger(logFn) { - let cwd = process.cwd(), - cwdRe = new RegExp("^[^/]*" + cwd.replace("/", "\\/") + "\/([^:]*:[0-9]*).*$"); - - function getErrorObject() { - try { - throw Error(); - } catch (err) { - return err; - } - } - - let err = getErrorObject(), - caller_line = err.stack.split("\n")[4], - args = [caller_line.replace(cwdRe, "$1 -")]; - - /* arguments.unshift() doesn't exist... */ - for (var i = 1; i < arguments.length; i++) { - args.push(arguments[i]); - } - - logFn.apply(this, args); -} - -console.log = lineLogger.bind(console, console.log); -console.warn = lineLogger.bind(console, console.warn); -console.error = lineLogger.bind(console, console.error); - diff --git a/server/scanner.js b/server/scanner.js index 46194ff..8ab1946 100755 --- a/server/scanner.js +++ b/server/scanner.js @@ -1,11 +1,29 @@ +/** + * scanner + * + * Face recognition: + * 1. For each photo, extract all faces. Store face rectangles. + * face_id unique + * photo_id foreign key + * top left bottom right + * identity_id + * distance (0 == truth; manually assigned identity) + * 2. For each face_id, create: + * normalized_file + * original_file + * 128 float + */ "use strict"; +/* meta directories are not scanned for photos */ +const metaDirectories = [ "thumbs", "raw", "face-data", ".git", "corrupt" ]; + const Promise = require("bluebird"), fs = require("fs"), config = require("config"), moment = require("moment"), - crypto = require("crypto"); - + crypto = require("crypto"), + { stat, mkdir, exists } = require("./lib/util"); let photoDB = null; @@ -42,63 +60,6 @@ let processRunning = false; const { spawn } = require('child_process'); const sharp = require("sharp"), exif = require("exif-reader"); -const stat = function (_path) { - if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) { - _path = _path.substring(picturesPath.length); - } - - let path = picturesPath + _path; - - return new Promise(function (resolve, reject) { - fs.stat(path, function (error, stats) { - if (error) { - return reject(error); - } - return resolve(stats); - }); - }); -} - -const mkdir = function (_path) { - if (_path.indexOf(picturesPath) == 0) { - _path = _path.substring(picturesPath.length); - } - - let parts = _path.split("/"), path; - - parts.unshift(picturesPath); - return Promise.mapSeries(parts, function (part) { - if (!path) { - path = picturesPath.replace(/\/$/, ""); - } else { - path += "/" + part; - } - - return stat(path).catch(function (error) { - if (error.code != "ENOENT") { - throw error; - } - - return new Promise(function (resolve, reject) { - fs.mkdir(path, function (error) { - if (error) { - return reject(error); - } - - return resolve(); - }); - }); - }); - }); -} - -const exists = function(path) { - return stat(path).then(function() { - return true; - }).catch(function() { - return false; - }); -} function convertRawToJpg(path, file) { setStatus("Converting " + path + file); @@ -464,8 +425,8 @@ function scanDir(parent, path) { return resolve([]); } - /* Remove 'thumbs' and 'raw' directories from being processed */ - files = files.filter(function(file) { + /* Remove meta-data directories from being processed */ + files = files.filter((file) => { for (var i = 0; i < files.length; i++) { /* If this file has an original NEF/ORF on the system, don't add the JPG to the DB */ if (rawExtension.exec(files[i]) && file == files[i].replace(rawExtension, ".jpg")) { @@ -480,8 +441,7 @@ function scanDir(parent, path) { return false; } } - - return file != "raw" && file != "thumbs" && file != ".git" && file != "corrupt"; + return metaDirectories.indexOf(file) == -1; }); return resolve(files);