diff --git a/server/db/photos.js b/server/db/photos.js index ef0c409..4239f74 100755 --- a/server/db/photos.js +++ b/server/db/photos.js @@ -64,6 +64,10 @@ function init() { type: Sequelize.BOOLEAN, defaultValue: 0 }, + deleted: { + type: Sequelize.BOOLEAN, + defaultValue: 0 + }, albumId: { type: Sequelize.INTEGER, allowNull: true, diff --git a/server/routes/photos.js b/server/routes/photos.js index d758de0..8619861 100755 --- a/server/routes/photos.js +++ b/server/routes/photos.js @@ -46,7 +46,7 @@ router.get("/memories/*", function(req, res/*, next*/) { let date = new Date(decodeURI(req.url).replace(/\?.*$/, "")); let query = "SELECT photos.*,albums.path AS path FROM photos " + "INNER JOIN albums ON (albums.id=photos.albumId) " + - "WHERE (photos.duplicate=0 AND photos.scanned NOT NULL AND " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1); + "WHERE (photos.duplicate=0 AND photos.deleted=0 AND photos.scanned NOT NULL AND " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1); // console.log("Memories for " + date.toISOString().replace(/T.*/, "")); // console.log(query); @@ -117,7 +117,7 @@ router.get("/*", function(req, res/*, next*/) { let path = decodeURI(req.url).replace(/\?.*$/, "").replace(/^\//, ""), query = "SELECT photos.*,albums.path AS path FROM photos " + "INNER JOIN albums ON (albums.id=photos.albumId AND albums.path LIKE :path) " + - "WHERE (photos.duplicate=0 AND photos.scanned NOT NULL " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1), + "WHERE (photos.duplicate=0 AND photos.scanned AND photos.deleted=0 NOT NULL " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1), replacements = { cursor: cursor, path: path + "%" diff --git a/server/scanner.js b/server/scanner.js index a98b54f..f9796d3 100755 --- a/server/scanner.js +++ b/server/scanner.js @@ -13,7 +13,7 @@ let photoDB = null; const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/"; -let processQueue = [], triedClean = []; +let processQueue = [], triedClean = [], lastScan; //const rawExtension = /\.(nef|orf)$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef", "orf" ]; @@ -70,7 +70,6 @@ const mkdir = function (_path) { } let parts = _path.split("/"), path; - parts.unshift(picturesPath); return Promise.mapSeries(parts, function (part) { @@ -188,20 +187,18 @@ function moveCorrupt(path, file) { /*******************************************************************************************************/ function processBlock(items) { - /* Invoke once per second to check if there are items to process */ - setTimeout(processBlock, 1000); - + if (items) { - console.log("Adding " + items.length + " on top of " + processQueue.length + " assets."); processQueue = processQueue.concat(items); } - if (processRunning || processQueue.length == 0) { + if (processRunning) { + /* Invoke once per second to check if there are items to process */ + setTimeout(processBlock, 1000); return; } - let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0), - needsProcessing = [], duplicates = []; + let processing = processQueue.splice(0), needsProcessing = [], duplicates = []; processRunning = true; @@ -215,9 +212,13 @@ function processBlock(items) { asset.hash = hash; }); }).then(function() { + let toProcess = processing.length, lastMessage = moment(); /* Needs to be one at a time in case there are multiple HASH collisions */ return Promise.mapSeries(processing, function(asset) { - return photoDB.sequelize.query("SELECT * FROM photohashes WHERE hash=:hash OR photoId=:id", { + return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " + + "LEFT JOIN photos ON (photos.id=photohashes.photoId) " + + "LEFT JOIN albums ON (albums.id=photos.albumId) " + + "WHERE hash=:hash OR photoId=:id", { replacements: asset, type: photoDB.sequelize.QueryTypes.SELECT }).then(function(results) { @@ -227,7 +228,9 @@ function processBlock(items) { } else if (results[0].hash != asset.hash) { query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)"; } else if (results[0].photoId != asset.id) { - console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename); + console.log("Duplicate asset: " + + "'" + asset.album.path + asset.filename + "' is a copy of " + + "'" + results[0].path + results[0].filename + "'"); duplicates.push(asset); return; } @@ -242,9 +245,16 @@ function processBlock(items) { return photoDB.sequelize.query(query, { replacements: asset }); + }).then(function() { + toProcess--; + if (moment().add(-5, 'seconds') > lastMessage) { + console.log("Hash items to be processed: " + toProcess); + lastMessage = moment(); + } }); }); }).then(function() { + let toProcess = processing.length, lastMessage = moment(); console.log(needsProcessing.length + " assets need to have metadata extracted"); return Promise.map(needsProcessing, function(asset) { var path = asset.album.path, @@ -382,7 +392,7 @@ function processBlock(items) { console.warn("Adding " + path + file + " back onto processing queue."); triedClean.push(path + file); - processQueue.push([ path, file, created, albumId ]); + processBlock([ path, file, created, albumId ]); }); }); }).catch(function() { @@ -401,11 +411,50 @@ function processBlock(items) { } }); - return photoDB.sequelize.query("UPDATE photos SET duplicate=1,scanned=CURRENT_TIME WHERE id IN (:dups)", { + if (dups.length == 0) { + return; + } + + return photoDB.sequelize.query("UPDATE photos SET duplicate=1,scanned=CURRENT_TIMESTAMP WHERE id IN (:dups)", { replacements: { dups: dups } }); + }).then(function() { + console.log("Looking for removed assets"); + return photoDB.sequelize.query("SELECT photos.scanned,photos.id,photos.filename,albums.path FROM photos " + + "LEFT JOIN albums ON (albums.id=photos.albumId) " + + "WHERE photos.deleted=0 AND DATETIME(photos.scanned)