ketr.photos/server/scanner.js

"use strict";

const Promise = require("bluebird"),
  fs = require("fs"),
  config = require("config"),
  moment = require("moment");

let scanning = 0;

let photoDB = null;

const picturesPath = config.get("picturesPath");

const processQueue = [];

function scanDir(parent, path) {
  let extensions = [ "jpg", "jpeg", "png", "gif" ],
    re = "\.((" + extensions.join(")|(") + "))$";
  re = new RegExp(re, "i");

  return photoDB.sequelize.query("SELECT id FROM albums WHERE path=:path AND parentId=:parent", {
    replacements: {
      path: path,
      parent: parent || null
    },
    type: photoDB.sequelize.QueryTypes.SELECT
  }).then(function(results) {
    if (results.length == 0) {
      console.log("Adding " + path + " under " + parent);
      return photoDB.sequelize.query("INSERT INTO albums SET path=:path,parentId=:parent", {
        replacements: {
          path: path,
          parent: parent || null
        },
      }).then(function(results) {
        return results[0];
      });
    } else {
      return results[0].id;
    }
  }).then(function(parent) {
    return new Promise(function(resolve, reject) {
    //    console.log("Scanning path " + path);

      fs.readdir(path, function(err, files) {
        if (err) {
          console.warn(" Could not readdir " + path);
          return resolve(null);
        }

        scanning++;

        let hasThumbs = false;
        for (let i = 0; i < files.length; i++) {
          if (files[i] == "thumbs") {
            hasThumbs = true;
            break;
          }
        }

        let tmp = Promise.resolve();

        if (!hasThumbs) {
          tmp = new Promise(function(resolve, reject) {
            fs.mkdir(path + "/thumbs", function(err) {
              if (err) {
                return reject("Unable to create " + paths + "/thumbs");
              }
              return resolve();
            });
          });
        }

        return tmp.then(function() {
          return Promise.map(files, function(file) {
            let filepath = path + "/" + file;
            if (file == "thumbs") {
              return resolve(true);
            }

            return new Promise(function(resolve, reject) {
              fs.stat(filepath, function(err, stats) {
                if (err) {
                  console.warn("Could not stat " + filepath);
                  return resolve(false);
                }

                if (stats.isDirectory()) {
                  return scanDir(parent, filepath, stats).then(function(entry) {
                    return resolve(true);
                  });
                }

                /* stats.isFile()  */
                if (!re.exec(file)) {
                  return resolve(true);
                }
          
                const replacements = {
                  path: path.slice(picturesPath.length),
                  filename: file
                };

                return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", {
                  replacements: replacements,
                  type: photoDB.sequelize.QueryTypes.SELECT
                }).then(function(photo) {
                  if (photo.length == 0) {
                    processQueue.push([ replacements.path, file, stats.mtime, parent ]);
                  }
                  return resolve(true);
                });
              });
            });
          }, {
            concurrency: 10
          }).then(function() {
            scanning--;
            if (scanning == 0) {
              const endStamp = Date.now();
              console.log("Scanning completed in " + Math.round(((endStamp - startStamp))) + "ms. " + processQueue.length + " items to process.");
            }
          }).then(function() {
            return resolve();
          });
        });
      });
    });
  });
}

const startStamp = Date.now();

let processRunning = false;

const { spawn } = require('child_process');

const sharp = require("sharp"), exif = require("exif-reader");

function triggerWatcher() {
  setTimeout(triggerWatcher, 1000);

  if (!processRunning && processQueue.length) {
    processRunning = true;
    return Promise.map(processQueue, function(entry) {
      var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3],
        src = picturesPath + path + "/" + file,
        dst = picturesPath + path + "/thumbs/" + file,
        image = sharp(src);
//      console.log("Processing " + src);
      return image.metadata().then(function(metadata) {
        if (metadata.exif) {
          metadata.exif = exif(metadata.exif);
          delete metadata.exif.thumbnail;
          delete metadata.exif.image;
          for (var key in metadata.exif.exif) {
            if (Buffer.isBuffer(metadata.exif.exif[key])) {
              metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
            }
          }
        }

        let replacements = {
          albumId: albumId,
          path: path,
          filename: file,
          width: metadata.width,
          height: metadata.height,
          added: moment().format().replace(/T.*/, "")
        };

        if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
          metadata.exif.exif.DateTimeOriginal.setHours(0, 0, 0, 0);
          metadata.exif.exif.DateTimeOriginal = metadata.exif.exif.DateTimeOriginal.toISOString().replace(/T.*/, "");
//          console.log(metadata.exif.exif.DateTimeOriginal);
          replacements.taken = moment(metadata.exif.exif.DateTimeOriginal, "YYYY-MM-DD").format().replace(/T.*/, "");
          replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format().replace(/T.*/, "");
        } else {
//          console.log("Missing EXIF info for: " + file);
          //console.log(JSON.stringify(metadata.exif, null, 2));
          let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = replacements.added;
          let match = file.match(patterns);
          if (match) {
            date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
//            console.log("Constructed date: " + date);
          } else {
            date = moment(created).format();
//            console.log("Date from file: ", src, date);
          }
          replacements.taken = replacements.modified = date;
        }

        return image.resize(256, 256).toFile(dst).then(function() {
          return photoDB.sequelize.query("INSERT INTO photos " +
            "SET albumId=:albumId,path=:path,filename=:filename,added=DATE(:added),modified=DATE(:modified),taken=DATE(:taken),width=:width,height=:height", {
            replacements: replacements
          });
        }).catch(function(error) {
          console.log("Error resizing or writing " + src, error);
          return Promise.Reject();
        });
      });
    }, {
      concurrency: 1
    });
  }
}

module.exports = {
  scan: function (db) {
    photoDB = db;
    return scanDir(0, picturesPath).then(function() {
      triggerWatcher();
    });
  }
};