scanner::scan can now be called dynamically to re-scan

Signed-off-by: James Ketrenos <james_git@ketrenos.com>

server/scanner.js

@@ -11,7 +11,7 @@ let photoDB = null;
 
 const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
 
-let processQueue = [], triedClean = [], lastScan;
+let processQueue = [], triedClean = [], lastScan = new Date("1900-01-01");
 
 //const rawExtension = /\.(nef|orf)$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef", "orf" ];
 
@@ -220,7 +220,7 @@ function processBlock(items) {
         if (results.length == 0) {
           query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
         } else if (results[0].hash != asset.hash) {
-          query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
+          query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
         } else if (results[0].photoId != asset.id) {
           console.log("Duplicate asset: " + 
             "'" + asset.album.path + asset.filename + "' is a copy of " +
@@ -517,7 +517,8 @@ function scanDir(parent, path) {
           filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
           name: file.replace(/.[^.]*$/, ""),
           stats: {
-            mtime: stats.mtime
+            mtime: stats.mtime,
+            ctime: stats.ctime
           },
           album: album
         });
@@ -573,7 +574,7 @@ function findOrCreateDBAlbum(transaction, album) {
 }
 
 function findOrUpdateDBAsset(transaction, asset) {
-  let query = "SELECT id,scanned,modified FROM photos WHERE albumId=:albumId AND filename=:filename";
+  let query = "SELECT id,DATETIME(scanned) AS scanned,DATETIME(modified) AS modified FROM photos WHERE albumId=:albumId AND filename=:filename";
   if (!asset.album || !asset.album.id) {
     let error = "Asset being processed without an album";
     console.error(error);
@@ -595,8 +596,8 @@ function findOrUpdateDBAsset(transaction, asset) {
       });
     } else {
       asset.id = results[0].id;
-      asset.scanned = results[0].scanned;
+      asset.scanned = new Date(results[0].scanned);
-      asset.modified = results[0].modified;
+      asset.modified = new Date(results[0].modified);
     }
   }).then(function() {
     return asset;
@@ -625,34 +626,41 @@ function computeHash(filepath) {
   });
 }
 
-module.exports = {
+function doScan() {
-  scan: function (db) {
+  /* 1. Scan for all assets which will be managed by the system. readdir
-    photoDB = db;
+    * 2. Check if entry in DB. Check mod-time in DB vs. stats from #1
-    /* 1. Scan for all assets which will be managed by the system. readdir
+    *  - For albums
-     * 2. Check if entry in DB. Check mod-time in DB vs. stats from #1
+    *  - For assets
-     *  - For albums
+    * 3. If not in DB, or mod-time changed, queue for HASH CHECK
-     *  - For assets
+    * 
-     * 3. If not in DB, or mod-time changed, queue for HASH CHECK
+    * HASH CHECK
-     * 
+    * 1. Compute HASH
-     * HASH CHECK
+    * 2. Check for HASH in photohash -- skip?
-     * 1. Compute HASH
+    * 3. Check for and create thumbs/FILE thumbs/scaled/FILE
-     * 2. Check for HASH in photohash -- skip?
+    * 4. If necessary, create JPG from RAW
-     * 3. Check for and create thumbs/FILE thumbs/scaled/FILE
+    * 5. Update last-scanned date in DB for entry
-     * 4. If necessary, create JPG from RAW
+    * 6. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were 
-     * 5. Update last-scanned date in DB for entry
+    *    removed on disk)? Also purge from the HASH table.
-     * 6. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were 
+    */
-     *    removed on disk)? Also purge from the HASH table.
+  let initialized = Date.now();
-     */
+  let now = Date.now();
-    let initialized = Date.now();
+  let needsProcessing = [];
-    let now = Date.now();
-    let needsProcessing = [];
 
-    lastScan = new Date();
+  return photoDB.sequelize.query("SELECT max(scanned) AS scanned FROM photos", {
+    type: photoDB.sequelize.QueryTypes.SELECT
+  }).then(function(results) {
+    if (results[0].scanned == null) {
+      lastScan = new Date("1800-01-01");
+    } else {
+      lastScan = new Date(results[0].scanned);
+    }
+    console.log("Updating any asset newer than " + moment(lastScan).format());
+  }).then(function() {
     return scanDir(null, picturesPath).spread(function(albums, assets) {
       console.log("Found " + assets.length + " assets in " + albums.length + " albums after " + 
         ((Date.now() - now) / 1000) + "s");
       /* One at a time, in series, as the album[] array has parents first, then descendants.
-       * Operating in parallel could result in a child being searched for prior to the parent */
+      * Operating in parallel could result in a child being searched for prior to the parent */
       now = Date.now();
 
       let toProcess = albums.length, lastMessage = moment();
@@ -676,23 +684,40 @@ module.exports = {
         let processed = 0, start = Date.now(), last = 0, updateScanned = [], newEntries = 0;
         return photoDB.sequelize.transaction(function(transaction) {
           return Promise.map(assets, function(asset) {
-            return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
+            return new Promise(function(resolve, reject) {
-              if (!asset.scanned) {
+              /* If both mtime and ctime of the asset are older than the lastScan, 
-                newEntries++;
+              * skip this asset. */
-              }
+              if (lastScan != null && asset.stats.mtime < lastScan && asset.stats.ctime < lastScan) {
-              if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) {
-                needsProcessing.push(asset);
-              } else {
                 updateScanned.push(asset.id);
+                return resolve(asset);
               }
+
+              return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
+                if (!asset.scanned) {
+                  newEntries++;
+                }
+                console.log(typeof asset.scanned);
+                if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) {
+                  console.log("Process asset: " + asset.scanned);
+                  needsProcessing.push(asset);
+                } else {
+                  console.log("Update scanned tag: " + asset.scanned);
+                  updateScanned.push(asset.id);
+                }
+                return asset;
+              }).then(function(asset) {
+                return resolve(asset);
+              }).catch(function(error) {
+                return reject(error);
+              });
             }).then(function(asset) {
               processed++;
-  
+
               let elapsed = Date.now() - start;
               if (elapsed < 5000) {
                 return asset;
               }
-  
+
               let remaining = assets.length - processed, eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
               console.log(remaining + " assets remaining be verified/updated " +
                 "(" + newEntries + " new entries, " + (processed - newEntries) + " up-to-date so far). ETA " + eta + "s");
@@ -727,5 +752,13 @@ module.exports = {
     }).then(function() {
       console.log("Total time to initialize DB and all scans: " + ((Date.now() - initialized) / 1000) + "s");
     });
+  });
+}
+
+module.exports = {
+  scan: function (db) {
+    photoDB = db;
+    return doScan();
   }
 };
+