From 531c6db4fd3c261139d9263bfde43f53979849cd Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Tue, 25 Sep 2018 22:35:04 -0700 Subject: [PATCH] Ready to plumb in processBlock to do the thumbnailing Signed-off-by: James Ketrenos --- server/routes/photos.js | 2 +- server/scanner.js | 219 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 200 insertions(+), 21 deletions(-) diff --git a/server/routes/photos.js b/server/routes/photos.js index 5ac0f6e..0d7628e 100755 --- a/server/routes/photos.js +++ b/server/routes/photos.js @@ -113,7 +113,7 @@ router.get("/*", function(req, res/*, next*/) { } let path = decodeURI(req.url).replace(/\?.*$/, ""), - query = "SELECT * FROM photos WHERE path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1); + query = "SELECT photos.* FROM photos INNER JOIN albums ON albums.id=photos.albumId AND albums.path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1); console.log("Fetching from: " + path); diff --git a/server/scanner.js b/server/scanner.js index b4b8d02..5b55ef4 100755 --- a/server/scanner.js +++ b/server/scanner.js @@ -13,7 +13,7 @@ let photoDB = null; const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/"; -const processQueue = [], triedClean = []; +let processQueue = [], triedClean = []; //const rawExtension = /\.(nef|orf)$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef", "orf" ]; @@ -464,6 +464,193 @@ function triggerWatcher() { } } /*******************************************************************************************************/ +let processTimeout = null; +function processBlock(items) { + if (!processTimeout) { + processTimeout = setTimeout(processBlock, 1000); + } + processTimeout = null; + + if (items) { + processQueue = processQueue.concat(items); + } + + return; + /* 'needsProcessing' should only have their scanned stamp updated once thumbnails + * have been created. */ + console.log(needsProcessing.length + " had their HASH updated. Updating scanned stamp."); + return db.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:scanned)", { + replacements: { + scanned: needsProcessing + } + }); + + if (!processRunning && processQueue.length) { + let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0); + processRunning = true; +console.log(processQueue.length); + /* Sort to newest files to be processed first */ + processing.sort(function(a, b) { + return b[2] - a[2]; + }); + + return Promise.map(processing, function(entry) { + var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3]; + + let tmp = Promise.resolve(file); + /* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */ + if (rawExtension.exec(file)) { + tmp = exists(picturesPath + path + "/" + file.replace(rawExtension, ".jpg")).then(function(exist) { + if (exist) { + return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */ + } + + return mkdir(picturesPath + path + "/raw").then(function() { + return convertRawToJpg(path, file); + }).then(function() { + return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */ + }); + }); + } + + return tmp.then(function(file) { + var src = picturesPath + path + "/" + file, + image = sharp(src); + + return image.limitInputPixels(1073741824).metadata().then(function(metadata) { + if (metadata.exif) { + metadata.exif = exif(metadata.exif); + delete metadata.exif.thumbnail; + delete metadata.exif.image; + for (var key in metadata.exif.exif) { + if (Buffer.isBuffer(metadata.exif.exif[key])) { + metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]"; + } + } + } + + let replacements = { + albumId: albumId, + name: file.replace(/.[^.]*$/, ""), + path: path, + filename: file, + width: metadata.width, + height: metadata.height, + added: moment().format() + }; + + /* Ensure that top level images are placed into an album with a root path */ + replacements.path = replacements.path || "/"; + + if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) { + replacements.taken = moment(metadata.exif.exif.DateTimeOriginal).format(); + replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format(); + + if (replacements.taken == "Invalid date" || replacements.taken.replace(/T.*/, "") == "1899-11-30") { + console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif)); + replacements.taken = replacements.modified = moment(created).format(); + } + } else { + /* Attempt to infer the datestamp from the filename */ + let date = moment(created).format(); + + let match = file.match(/WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/); + if (match) { + date = moment((match[1]+" "+match[2]), "YYYY-MM-DD h.mm.ss a").format(); + if (date == "Invalid date") { + date = moment(created).format(); + } + } else { + match = file.match(/(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/); + if (match) { + if (match[2]) { /* Stamp had time in it */ + date = moment((match[1]+""+match[2]).replace(/-/g, ""), "YYYYMMDDHHmmss").format(); + } else { + date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format(); + } + if (date == "Invalid date") { + date = moment(created).format(); + } + } else { + date = moment(created).format(); + } + } + replacements.taken = replacements.modified = date; + } + + let dst = picturesPath + path + "/thumbs/" + file; + + return exists(dst).then(function(exist) { + if (exist) { + return; + } + + return image.resize(256, 256).toFile(dst).catch(function(error) { + console.error("Error resizing image: " + dst, error); + throw error; + }); + }).then(function() { + let dst = picturesPath + path + "/thumbs/scaled/" + file; + return exists(dst).then(function(exist) { + if (exist) { + return; + } + + return image.resize(Math.min(1024, metadata.width)).toFile(dst).catch(function(error) { + console.error("Error resizing image: " + dst, error); + throw error; + }); + }); + }).then(function() { + return photoDB.sequelize.query("INSERT INTO photos " + + "(albumId,path,filename,added,modified,taken,width,height,name)" + + "VALUES(:albumId,:path,:filename,DATETIME(:added),DATETIME(:modified),DATETIME(:taken),:width,:height,:name)", { + replacements: replacements + }); + }).then(function() { + toProcess--; + if (moment().add(-5, 'seconds') > lastMessage) { + console.log("Items to be processed: " + toProcess); + lastMessage = moment(); + } + }); + }).catch(function(error) { + console.error("Error reading image " + src + ": ", error); + return moveCorrupt(path, file).then(function() { + + /* If the original file was not a NEF/ORF, we are done... */ + if (!rawExtension.exec(entry[1])) { + return; + } + + /* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */ + for (var i = 0; i < triedClean.length; i++) { + if (triedClean[i] == path + "/" + file) { + /* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */ + // return moveCorrupt(path, entry[1]); + + console.error("Already attempted to convert NEF/ORF to JPG: " + path + "/" + file); + return; + } + } + + console.warn("Adding " + path + "/" + file + " back onto processing queue."); + triedClean.push(path + "/" + file); + processQueue.push([ path, file, created, albumId ]); + }); + }); + }).catch(function() { + console.warn("Continuing file processing."); + }); + }, { + concurrency: 1 + }).then(function() { + console.log("Completed processing queue."); + }); + } +} + + function scanDir(parent, path) { let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"), album = { @@ -650,6 +837,7 @@ module.exports = { */ let initialized = Date.now(); let now = Date.now(); + const needsProcessing = [], duplicates = []; return scanDir(null, picturesPath).spread(function(albums, assets) { console.log("Found " + assets.length + " assets in " + albums.length + " albums after " + ((Date.now() - now) / 1000) + "s"); @@ -679,7 +867,7 @@ module.exports = { } let remaining = assets.length - processed; - console.log(remaining + " assets remaining. ETA " + + console.log(remaining + " assets remaining to have DB entries updated. ETA " + Math.ceil((elapsed / 1000) * remaining / (processed - last)) + "s"); last = processed; start = Date.now(); @@ -693,9 +881,9 @@ module.exports = { asset.hash = hash; }); }).then(function() { - let needsProcessing = []; - return Promise.map(hashNeeded, function(asset) { - return db.sequelize.query("SELECT hash FROM photohashes WHERE photoId=:id", { + /* Needs to be one at a time in case there are multiple HASH collisions */ + return Promise.mapSeries(hashNeeded, function(asset) { + return db.sequelize.query("SELECT * FROM photohashes WHERE hash=:hash OR photoId=:id", { replacements: asset, type: photoDB.sequelize.QueryTypes.SELECT }).then(function(results) { @@ -704,6 +892,10 @@ module.exports = { query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; } else if (results[0].hash != asset.hash) { query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)"; + } else if (results[0].photoId != asset.id) { + console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename); + duplicates.push(asset); + return; } else { return; } @@ -715,29 +907,16 @@ module.exports = { needsProcessing.push(asset.id); }); }); - }, { - concurrency: 5 }).then(function() { - if (!needsProcessing.length) { - return; - } - - /* 'needsProcessing' should only have their scanned stamp updated once thumbnails - * have been created. */ - console.log(needsProcessing.length + " had their HASH updated. Updating scanned stamp."); - return db.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:scanned)", { - replacements: { - scanned: needsProcessing - } - }); + processBlock(needsProcessing); }); }); }).then(function() { console.log("Processed " + assets.length + " asset DB entries in " + ((Date.now() - now) / 1000) + "s"); + console.log(duplicates.length + " duplicates."); }); }); - /*triggerWatcher();*/ }).then(function() { console.log("Total time to initialize DB and all scans: " + ((Date.now() - initialized) / 1000) + "s"); });