Ready to plumb in processBlock to do the thumbnailing

Signed-off-by: James Ketrenos <james_git@ketrenos.com>

server/routes/photos.js

@@ -113,7 +113,7 @@ router.get("/*", function(req, res/*, next*/) {
   }
 
   let path = decodeURI(req.url).replace(/\?.*$/, ""),
-    query = "SELECT * FROM photos WHERE path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
+    query = "SELECT photos.* FROM photos INNER JOIN albums ON albums.id=photos.albumId AND albums.path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
 
   console.log("Fetching from: " + path);
   

server/scanner.js

@@ -13,7 +13,7 @@ let photoDB = null;
 
 const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
 
-const processQueue = [], triedClean = [];
+let processQueue = [], triedClean = [];
 
 //const rawExtension = /\.(nef|orf)$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef", "orf" ];
 
@@ -464,6 +464,193 @@ function triggerWatcher() {
   }
 }
 /*******************************************************************************************************/
+let processTimeout = null;
+function processBlock(items) {
+  if (!processTimeout) {
+    processTimeout = setTimeout(processBlock, 1000);
+  }
+  processTimeout = null;
+
+  if (items) {
+    processQueue = processQueue.concat(items);
+  }
+  
+  return;
+  /* 'needsProcessing' should only have their scanned stamp updated once thumbnails
+   * have been created. */
+  console.log(needsProcessing.length + " had their HASH updated. Updating scanned stamp.");
+  return db.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:scanned)", {
+    replacements: {
+      scanned: needsProcessing
+    }
+  });
+
+  if (!processRunning && processQueue.length) {
+    let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0);
+    processRunning = true;
+console.log(processQueue.length);
+    /* Sort to newest files to be processed first */
+    processing.sort(function(a, b) {
+      return b[2] - a[2];
+    });
+
+    return Promise.map(processing, function(entry) {
+      var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3];
+        
+      let tmp = Promise.resolve(file);
+      /* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */
+      if (rawExtension.exec(file)) {
+        tmp = exists(picturesPath + path + "/" + file.replace(rawExtension, ".jpg")).then(function(exist) {
+          if (exist) {
+            return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
+          }
+          
+          return mkdir(picturesPath + path + "/raw").then(function() {
+            return convertRawToJpg(path, file);
+          }).then(function() {
+            return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
+          });
+        });
+      }
+
+      return tmp.then(function(file) {
+        var src = picturesPath + path + "/" + file,
+          image = sharp(src);
+
+        return image.limitInputPixels(1073741824).metadata().then(function(metadata) {
+          if (metadata.exif) {
+            metadata.exif = exif(metadata.exif);
+            delete metadata.exif.thumbnail;
+            delete metadata.exif.image;
+            for (var key in metadata.exif.exif) {
+              if (Buffer.isBuffer(metadata.exif.exif[key])) {
+                metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
+              }
+            }
+          }
+
+          let replacements = {
+            albumId: albumId,
+            name: file.replace(/.[^.]*$/, ""),
+            path: path,
+            filename: file,
+            width: metadata.width,
+            height: metadata.height,
+            added: moment().format()
+          };
+
+          /* Ensure that top level images are placed into an album with a root path */
+          replacements.path = replacements.path || "/";
+
+          if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
+            replacements.taken = moment(metadata.exif.exif.DateTimeOriginal).format();
+            replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format();
+
+            if (replacements.taken == "Invalid date" || replacements.taken.replace(/T.*/, "") == "1899-11-30") {
+              console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
+              replacements.taken = replacements.modified = moment(created).format();
+            }
+          } else {
+            /* Attempt to infer the datestamp from the filename */
+            let date = moment(created).format();
+
+            let match = file.match(/WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/);
+            if (match) {
+              date = moment((match[1]+" "+match[2]), "YYYY-MM-DD h.mm.ss a").format();
+              if (date == "Invalid date") {
+                date = moment(created).format();
+              }
+            } else {
+              match = file.match(/(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/);
+              if (match) {
+                if (match[2]) { /* Stamp had time in it */
+                  date = moment((match[1]+""+match[2]).replace(/-/g, ""), "YYYYMMDDHHmmss").format();
+                } else {
+                  date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
+                }
+                if (date == "Invalid date") {
+                  date = moment(created).format();
+                }
+              } else {
+                date = moment(created).format();
+              }
+            }
+            replacements.taken = replacements.modified = date;
+          }
+
+          let dst = picturesPath + path + "/thumbs/" + file;
+
+          return exists(dst).then(function(exist) {
+            if (exist) {
+              return;
+            }
+
+            return image.resize(256, 256).toFile(dst).catch(function(error) {
+              console.error("Error resizing image: " + dst, error);
+              throw error;
+            });
+          }).then(function() {
+            let dst = picturesPath + path + "/thumbs/scaled/" + file;
+            return exists(dst).then(function(exist) {
+              if (exist) {
+                return;
+              }
+  
+              return image.resize(Math.min(1024, metadata.width)).toFile(dst).catch(function(error) {
+                console.error("Error resizing image: " + dst, error);
+                throw error;
+              });
+            });
+          }).then(function() {
+            return photoDB.sequelize.query("INSERT INTO photos " +
+              "(albumId,path,filename,added,modified,taken,width,height,name)" +
+              "VALUES(:albumId,:path,:filename,DATETIME(:added),DATETIME(:modified),DATETIME(:taken),:width,:height,:name)", {
+              replacements: replacements
+            });
+          }).then(function() {
+            toProcess--;
+            if (moment().add(-5, 'seconds') > lastMessage) {
+              console.log("Items to be processed: "  + toProcess);
+              lastMessage = moment();
+            }
+          });
+        }).catch(function(error) {
+          console.error("Error reading image " + src + ": ", error);
+          return moveCorrupt(path, file).then(function() {
+
+            /* If the original file was not a NEF/ORF, we are done... */
+            if (!rawExtension.exec(entry[1])) {
+              return;
+            }
+
+            /* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */
+            for (var i = 0; i < triedClean.length; i++) {
+              if (triedClean[i] == path + "/" + file) {
+                /* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */
+                // return moveCorrupt(path, entry[1]);
+
+                console.error("Already attempted to convert NEF/ORF to JPG: " + path + "/" + file);
+                return;
+              }
+            }
+
+            console.warn("Adding " + path + "/" + file + " back onto processing queue.");
+            triedClean.push(path + "/" + file);
+            processQueue.push([ path, file, created, albumId ]);
+          });
+        });
+      }).catch(function() {
+        console.warn("Continuing file processing.");
+      });
+    }, {
+      concurrency: 1
+    }).then(function() {
+      console.log("Completed processing queue.");
+    });
+  }
+}
+
+
 function scanDir(parent, path) {
   let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
     album = {
@@ -650,6 +837,7 @@ module.exports = {
      */
     let initialized = Date.now();
     let now = Date.now();
+    const needsProcessing = [], duplicates = [];
     return scanDir(null, picturesPath).spread(function(albums, assets) {
       console.log("Found " + assets.length + " assets in " + albums.length + " albums after " + 
         ((Date.now() - now) / 1000) + "s");
@@ -679,7 +867,7 @@ module.exports = {
             }
 
             let remaining = assets.length - processed;
-            console.log(remaining + " assets remaining. ETA " + 
+            console.log(remaining + " assets remaining to have DB entries updated. ETA " + 
               Math.ceil((elapsed / 1000) * remaining / (processed - last)) + "s");
             last = processed;
             start = Date.now();
@@ -693,9 +881,9 @@ module.exports = {
               asset.hash = hash;
             });
           }).then(function() {
-            let needsProcessing = [];
+            /* Needs to be one at a time in case there are multiple HASH collisions */
-            return Promise.map(hashNeeded, function(asset) {
+            return Promise.mapSeries(hashNeeded, function(asset) {
-              return db.sequelize.query("SELECT hash FROM photohashes WHERE photoId=:id", {
+              return db.sequelize.query("SELECT * FROM photohashes WHERE hash=:hash OR photoId=:id", {
                 replacements: asset,
                 type: photoDB.sequelize.QueryTypes.SELECT
               }).then(function(results) {
@@ -704,6 +892,10 @@ module.exports = {
                   query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
                 } else if (results[0].hash != asset.hash) {
                   query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
+                } else if (results[0].photoId != asset.id) {
+                  console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename);
+                  duplicates.push(asset);
+                  return;
                 } else {
                   return;
                 }
@@ -715,29 +907,16 @@ module.exports = {
                   needsProcessing.push(asset.id);
                 });
               });
-            }, {
-              concurrency: 5
             }).then(function() {
-              if (!needsProcessing.length) {
+              processBlock(needsProcessing);
-                return;
-              }
-
-              /* 'needsProcessing' should only have their scanned stamp updated once thumbnails
-               * have been created. */
-              console.log(needsProcessing.length + " had their HASH updated. Updating scanned stamp.");
-              return db.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:scanned)", {
-                replacements: {
-                  scanned: needsProcessing
-                }
-              });
             });
           });
         }).then(function() {
           console.log("Processed " + assets.length + " asset DB entries in " + 
             ((Date.now() - now) / 1000) + "s");
+          console.log(duplicates.length + " duplicates.");
         });
       });
-      /*triggerWatcher();*/
     }).then(function() {
       console.log("Total time to initialize DB and all scans: " + ((Date.now() - initialized) / 1000) + "s");
     });