Track duplicates, removed, and added items

Signed-off-by: James Ketrenos <james_git@ketrenos.com>

server/db/photos.js

@@ -64,6 +64,10 @@ function init() {
         type: Sequelize.BOOLEAN,
         defaultValue: 0
       },
+      deleted: {
+        type: Sequelize.BOOLEAN,
+        defaultValue: 0
+      },
       albumId: {
         type: Sequelize.INTEGER,
         allowNull: true,

server/routes/photos.js

@@ -46,7 +46,7 @@ router.get("/memories/*", function(req, res/*, next*/) {
   let date = new Date(decodeURI(req.url).replace(/\?.*$/, ""));
   let query = "SELECT photos.*,albums.path AS path FROM photos " +
     "INNER JOIN albums ON (albums.id=photos.albumId) " +
-    "WHERE (photos.duplicate=0 AND photos.scanned NOT NULL AND " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
+    "WHERE (photos.duplicate=0 AND photos.deleted=0 AND photos.scanned NOT NULL AND " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
 
 //  console.log("Memories for " + date.toISOString().replace(/T.*/, ""));
 //  console.log(query);
@@ -117,7 +117,7 @@ router.get("/*", function(req, res/*, next*/) {
   let path = decodeURI(req.url).replace(/\?.*$/, "").replace(/^\//, ""),
     query = "SELECT photos.*,albums.path AS path FROM photos " +
       "INNER JOIN albums ON (albums.id=photos.albumId AND albums.path LIKE :path) " +
-      "WHERE (photos.duplicate=0 AND photos.scanned NOT NULL " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1),
+      "WHERE (photos.duplicate=0 AND photos.scanned AND photos.deleted=0 NOT NULL " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1),
       replacements = {
         cursor: cursor,
         path: path + "%"

server/scanner.js

@@ -13,7 +13,7 @@ let photoDB = null;
 
 const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
 
-let processQueue = [], triedClean = [];
+let processQueue = [], triedClean = [], lastScan;
 
 //const rawExtension = /\.(nef|orf)$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef", "orf" ];
 
@@ -71,7 +71,6 @@ const mkdir = function (_path) {
 
   let parts = _path.split("/"), path;
   
-  
   parts.unshift(picturesPath);
   return Promise.mapSeries(parts, function (part) {
     if (!path) {
@@ -188,20 +187,18 @@ function moveCorrupt(path, file) {
 
 /*******************************************************************************************************/
 function processBlock(items) {
-  /* Invoke once per second to check if there are items to process */
-  setTimeout(processBlock, 1000);
   
   if (items) {
-    console.log("Adding " + items.length + " on top of " + processQueue.length + " assets.");
     processQueue = processQueue.concat(items);
   }
   
-  if (processRunning || processQueue.length == 0) {
+  if (processRunning) {
+    /* Invoke once per second to check if there are items to process */
+    setTimeout(processBlock, 1000);
     return;
   }
 
-  let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0),
+  let processing = processQueue.splice(0), needsProcessing = [], duplicates = [];
-    needsProcessing = [], duplicates = [];
 
   processRunning = true;
 
@@ -215,9 +212,13 @@ function processBlock(items) {
       asset.hash = hash;
     });
   }).then(function() {
+    let toProcess = processing.length, lastMessage = moment();
     /* Needs to be one at a time in case there are multiple HASH collisions */
     return Promise.mapSeries(processing, function(asset) {
-      return photoDB.sequelize.query("SELECT * FROM photohashes WHERE hash=:hash OR photoId=:id", {
+      return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
+        "LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
+        "LEFT JOIN albums ON (albums.id=photos.albumId) " +
+        "WHERE hash=:hash OR photoId=:id", {
         replacements: asset,
         type: photoDB.sequelize.QueryTypes.SELECT
       }).then(function(results) {
@@ -227,7 +228,9 @@ function processBlock(items) {
         } else if (results[0].hash != asset.hash) {
           query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
         } else if (results[0].photoId != asset.id) {
-          console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename);
+          console.log("Duplicate asset: " + 
+            "'" + asset.album.path + asset.filename + "' is a copy of " +
+            "'" + results[0].path + results[0].filename + "'");
           duplicates.push(asset);
           return;
         }
@@ -242,9 +245,16 @@ function processBlock(items) {
         return photoDB.sequelize.query(query, {
           replacements: asset
         });
+      }).then(function() {
+        toProcess--;
+        if (moment().add(-5, 'seconds') > lastMessage) {
+          console.log("Hash items to be processed: "  + toProcess);
+          lastMessage = moment();
+        }
       });
     });
   }).then(function() {
+    let toProcess = processing.length, lastMessage = moment();
     console.log(needsProcessing.length + " assets need to have metadata extracted");
     return Promise.map(needsProcessing, function(asset) {
       var path = asset.album.path,
@@ -382,7 +392,7 @@ function processBlock(items) {
 
             console.warn("Adding " + path + file + " back onto processing queue.");
             triedClean.push(path + file);
-            processQueue.push([ path, file, created, albumId ]);
+            processBlock([ path, file, created, albumId ]);
           });
         });
       }).catch(function() {
@@ -401,11 +411,50 @@ function processBlock(items) {
       }
     });
 
-    return photoDB.sequelize.query("UPDATE photos SET duplicate=1,scanned=CURRENT_TIME WHERE id IN (:dups)", {
+    if (dups.length == 0) {
+      return;
+    }
+
+    return photoDB.sequelize.query("UPDATE photos SET duplicate=1,scanned=CURRENT_TIMESTAMP WHERE id IN (:dups)", {
       replacements: {
         dups: dups
       }
     });
+  }).then(function() {
+    console.log("Looking for removed assets");
+    return photoDB.sequelize.query("SELECT photos.scanned,photos.id,photos.filename,albums.path FROM photos " +
+        "LEFT JOIN albums ON (albums.id=photos.albumId) " +
+        "WHERE photos.deleted=0 AND DATETIME(photos.scanned)<DATETIME(:lastScan)", {
+      replacements: {
+        lastScan: lastScan
+      },
+      type: photoDB.sequelize.QueryTypes.SELECT
+    }).then(function(results) {
+      let deleted = [];
+      console.log("Checking " + results.length + " assets to see if they are on disk.");
+      return Promise.map(results, function(asset) {
+        return exists(asset.path + asset.filename).then(function(exist) {
+          if (!exist) {
+            console.log(asset.path + asset.filename + " no longer exists on disk. Marking as deleted.");
+            deleted.push(asset.id);
+          }
+        });
+      }).then(function() {
+        return photoDB.sequelize.query("UPDATE photos SET deleted=1,scanned=CURRENT_TIMESTAMP WHERE id IN (:deleted)", {
+          replacements: {
+            deleted: deleted
+          }
+        }).then(function() {
+          return photoDB.sequelize.query("DELETE FROM photohashes WHERE photoId IN (:deleted)", {
+            replacements: {
+              deleted: deleted
+            }
+          });
+        }).then(function() {
+          console.log(deleted.length + " assets deleted.");
+        });
+      });
+    });
   }).then(function() {
     processRunning = false;
   });
@@ -516,7 +565,6 @@ function findOrCreateDBAlbum(album) {
       if (!album.parent) {
         console.warn("Creating top level album: " + picturesPath);
       }
-      console.log("album: " + album.path);
       return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
         replacements: album
       }).spread(function(results, metadata) {
@@ -544,11 +592,6 @@ function findOrUpdateDBAsset(asset) {
     type: photoDB.sequelize.QueryTypes.SELECT
   }).then(function(results) {
     if (results.length == 0) {
-      /* Not in DB at all; HASH needs to be created. We could use the sharp loader to create
-       * a buffer, then pass that around. It would complicate our code, so instead we will
-       * leverage the OS' ability to cache read-only file assets in the cache and to pass
-       * those to the sharp layer later :) Accordingly, we will load the file here, create a SHA
-       * and create that HASH entry (unless it already exists, in which case DUPLICATE!!!! Gah.) */
       return photoDB.sequelize.query("INSERT INTO photos " +
         "(albumId,filename,name) " +
         "VALUES(:albumId,:filename,:name)", {
@@ -611,6 +654,8 @@ module.exports = {
     let initialized = Date.now();
     let now = Date.now();
     const needsProcessing = [];
+
+    lastScan = new Date();
     return scanDir(null, picturesPath).spread(function(albums, assets) {
       console.log("Found " + assets.length + " assets in " + albums.length + " albums after " + 
         ((Date.now() - now) / 1000) + "s");
@@ -625,11 +670,13 @@ module.exports = {
           ((Date.now() - now) / 1000) + "s");
         now = Date.now();
 
-        let processed = 0, start = Date.now(), last = 0;
+        let processed = 0, start = Date.now(), last = 0, updateScanned = [];
         return Promise.map(assets, function(asset) {
           return findOrUpdateDBAsset(asset).then(function(asset) {
             if (asset.scanned < asset.stats.mtime) {
               needsProcessing.push(asset);
+            } else {
+              updateScanned.push(asset.id);
             }
           }).then(function(asset) {
             processed++;
@@ -647,6 +694,16 @@ module.exports = {
           });
         }, {
           concurrency: 5
+        }).then(function() {
+          if (updateScanned.length) {
+            return photoDB.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:ids)", {
+              replacements: {
+                ids: updateScanned
+              }
+            }).then(function() {
+              console.log("Updated scan date of " + updateScanned.length + " assets");
+            });
+          }
         }).then(function() {
           console.log(needsProcessing.length + " assets need HASH computed");
           processBlock(needsProcessing);