diff --git a/server/scanner.js b/server/scanner.js index ecefc7e..55a1853 100755 --- a/server/scanner.js +++ b/server/scanner.js @@ -11,7 +11,7 @@ let photoDB = null; const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/"; -let processQueue = [], triedClean = [], lastScan; +let processQueue = [], triedClean = [], lastScan = new Date("1900-01-01"); //const rawExtension = /\.(nef|orf)$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef", "orf" ]; @@ -220,7 +220,7 @@ function processBlock(items) { if (results.length == 0) { query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; } else if (results[0].hash != asset.hash) { - query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)"; + query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id"; } else if (results[0].photoId != asset.id) { console.log("Duplicate asset: " + "'" + asset.album.path + asset.filename + "' is a copy of " + @@ -517,7 +517,8 @@ function scanDir(parent, path) { filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */ name: file.replace(/.[^.]*$/, ""), stats: { - mtime: stats.mtime + mtime: stats.mtime, + ctime: stats.ctime }, album: album }); @@ -573,7 +574,7 @@ function findOrCreateDBAlbum(transaction, album) { } function findOrUpdateDBAsset(transaction, asset) { - let query = "SELECT id,scanned,modified FROM photos WHERE albumId=:albumId AND filename=:filename"; + let query = "SELECT id,DATETIME(scanned) AS scanned,DATETIME(modified) AS modified FROM photos WHERE albumId=:albumId AND filename=:filename"; if (!asset.album || !asset.album.id) { let error = "Asset being processed without an album"; console.error(error); @@ -595,8 +596,8 @@ function findOrUpdateDBAsset(transaction, asset) { }); } else { asset.id = results[0].id; - asset.scanned = results[0].scanned; - asset.modified = results[0].modified; + asset.scanned = new Date(results[0].scanned); + asset.modified = new Date(results[0].modified); } }).then(function() { return asset; @@ -625,34 +626,41 @@ function computeHash(filepath) { }); } -module.exports = { - scan: function (db) { - photoDB = db; - /* 1. Scan for all assets which will be managed by the system. readdir - * 2. Check if entry in DB. Check mod-time in DB vs. stats from #1 - * - For albums - * - For assets - * 3. If not in DB, or mod-time changed, queue for HASH CHECK - * - * HASH CHECK - * 1. Compute HASH - * 2. Check for HASH in photohash -- skip? - * 3. Check for and create thumbs/FILE thumbs/scaled/FILE - * 4. If necessary, create JPG from RAW - * 5. Update last-scanned date in DB for entry - * 6. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were - * removed on disk)? Also purge from the HASH table. - */ - let initialized = Date.now(); - let now = Date.now(); - let needsProcessing = []; +function doScan() { + /* 1. Scan for all assets which will be managed by the system. readdir + * 2. Check if entry in DB. Check mod-time in DB vs. stats from #1 + * - For albums + * - For assets + * 3. If not in DB, or mod-time changed, queue for HASH CHECK + * + * HASH CHECK + * 1. Compute HASH + * 2. Check for HASH in photohash -- skip? + * 3. Check for and create thumbs/FILE thumbs/scaled/FILE + * 4. If necessary, create JPG from RAW + * 5. Update last-scanned date in DB for entry + * 6. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were + * removed on disk)? Also purge from the HASH table. + */ + let initialized = Date.now(); + let now = Date.now(); + let needsProcessing = []; - lastScan = new Date(); + return photoDB.sequelize.query("SELECT max(scanned) AS scanned FROM photos", { + type: photoDB.sequelize.QueryTypes.SELECT + }).then(function(results) { + if (results[0].scanned == null) { + lastScan = new Date("1800-01-01"); + } else { + lastScan = new Date(results[0].scanned); + } + console.log("Updating any asset newer than " + moment(lastScan).format()); + }).then(function() { return scanDir(null, picturesPath).spread(function(albums, assets) { console.log("Found " + assets.length + " assets in " + albums.length + " albums after " + ((Date.now() - now) / 1000) + "s"); /* One at a time, in series, as the album[] array has parents first, then descendants. - * Operating in parallel could result in a child being searched for prior to the parent */ + * Operating in parallel could result in a child being searched for prior to the parent */ now = Date.now(); let toProcess = albums.length, lastMessage = moment(); @@ -676,23 +684,40 @@ module.exports = { let processed = 0, start = Date.now(), last = 0, updateScanned = [], newEntries = 0; return photoDB.sequelize.transaction(function(transaction) { return Promise.map(assets, function(asset) { - return findOrUpdateDBAsset(transaction, asset).then(function(asset) { - if (!asset.scanned) { - newEntries++; - } - if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) { - needsProcessing.push(asset); - } else { + return new Promise(function(resolve, reject) { + /* If both mtime and ctime of the asset are older than the lastScan, + * skip this asset. */ + if (lastScan != null && asset.stats.mtime < lastScan && asset.stats.ctime < lastScan) { updateScanned.push(asset.id); + return resolve(asset); } + + return findOrUpdateDBAsset(transaction, asset).then(function(asset) { + if (!asset.scanned) { + newEntries++; + } + console.log(typeof asset.scanned); + if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) { + console.log("Process asset: " + asset.scanned); + needsProcessing.push(asset); + } else { + console.log("Update scanned tag: " + asset.scanned); + updateScanned.push(asset.id); + } + return asset; + }).then(function(asset) { + return resolve(asset); + }).catch(function(error) { + return reject(error); + }); }).then(function(asset) { processed++; - + let elapsed = Date.now() - start; if (elapsed < 5000) { return asset; } - + let remaining = assets.length - processed, eta = Math.ceil((elapsed / 1000) * remaining / (processed - last)); console.log(remaining + " assets remaining be verified/updated " + "(" + newEntries + " new entries, " + (processed - newEntries) + " up-to-date so far). ETA " + eta + "s"); @@ -727,5 +752,13 @@ module.exports = { }).then(function() { console.log("Total time to initialize DB and all scans: " + ((Date.now() - initialized) / 1000) + "s"); }); + }); +} + +module.exports = { + scan: function (db) { + photoDB = db; + return doScan(); } }; +