From ae839e440ba2418c202e3e8614a25395654b5b04 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Wed, 26 Sep 2018 15:53:47 -0700 Subject: [PATCH] Working on path structure re-creation Signed-off-by: James Ketrenos --- server/scanner.js | 420 +++++++--------------------------------------- 1 file changed, 65 insertions(+), 355 deletions(-) diff --git a/server/scanner.js b/server/scanner.js index 5b55ef4..142325a 100755 --- a/server/scanner.js +++ b/server/scanner.js @@ -20,11 +20,11 @@ let processQueue = [], triedClean = []; const rawExtension = /\.nef$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef" ]; function removeNewerFile(path, fileA, fileB) { - fs.stat(path + "/" + fileA, function(err, statsA) { + fs.stat(path + fileA, function(err, statsA) { if (err) { return; } - fs.stat(path + "/" + fileB, function(err, statsB) { + fs.stat(path + fileB, function(err, statsB) { if (err) { return; } @@ -39,122 +39,6 @@ function removeNewerFile(path, fileA, fileB) { }); } -function _scanDir(parent, path) { - let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"), - replacements = { - path: path.slice(picturesPath.length), - name: path.replace(/.*\//, "").replace(/_/g, " "), - parent: parent || null - }; - - /* Ensure that top level images are placed into an album with a root path */ - replacements.path = replacements.path || "/"; - - let query = "SELECT id FROM albums WHERE path=:path AND "; - if (!parent) { - query += "parentId IS NULL"; - } else { - query += "parentId=:parent"; - } - - return photoDB.sequelize.query(query, { - replacements: replacements, - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(results) { - if (results.length == 0) { -// console.log("Adding " + path + " under " + parent, replacements); - return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parent,:name)", { - replacements: replacements - }).spread(function(results, metadata) { - return metadata.lastID; - }); - } else { - return results[0].id; - } - }).then(function(parent) { - return new Promise(function(resolve, reject) { - console.log("Scanning ..." + replacements.path); - - fs.readdir(path, function(err, files) { - if (err) { - console.warn("Could not readdir " + path); - return resolve([]); - } - - return resolve(files); - }); - }).then(function(files) { - scanning++; - - /* Remove 'thumbs' and 'raw' directories from being processed */ - files = files.filter(function(file) { - for (var i = 0; i < files.length; i++) { - /* If this file has an original NEF/ORF on the system, don't add the JPG to the DB */ - if (rawExtension.exec(files[i]) && file == files[i].replace(rawExtension, ".jpg")) { - return false; - } - - /* If there is a different CASE (eg. JPG vs jpg) don't add it, and remove the 'lower case' - * version from disk. */ - if (file != files[i] && file.toUpperCase() == files[i]) { - removeNewerFile(path, file, files[i]); - console.log("Duplicate file in " + path + ": ", file, files[i]); - return false; - } - } - - return file != "raw" && file != "thumbs" && file != ".git" && file != "corrupt"; - }); - - return mkdir(path + "/thumbs/scaled").then(function() { - return Promise.map(files, function(file) { - let filepath = path + "/" + file; - - return stat(filepath).then(function(stats) { - if (stats.isDirectory()) { - return scanDir(parent, filepath).catch(function(error) { - console.warn("Could not scanDir " + filepath + ": " + error); - }); - } - - /* Check file extensions */ - if (!re.exec(file)) { - return; - } - - const replacements = { - path: path.slice(picturesPath.length), - filename: file.replace(rawExtension, ".jpg") /* We will be converting from NEF/ORF => JPG */ - }; - - replacements.path = replacements.path || "/"; - - return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", { - replacements: replacements, - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(photo) { - if (photo.length == 0) { - processQueue.push([ replacements.path, file, stats.mtime, parent ]); - } - }).catch(function(error) { - console.error("Sequelize.query failed: ", error); - }); - }).catch(function(error) { - console.warn("Could not stat " + filepath + ": " + error); - }); - }, { - concurrency: 1 - }).then(function() { - scanning--; - if (scanning == 0) { - const endStamp = Date.now(); - console.log("Scanning completed in " + Math.round(((endStamp - startStamp))) + "ms. " + processQueue.length + " items to process."); - } - }); - }); - }); - }); -} const startStamp = Date.now(); @@ -179,14 +63,17 @@ const stat = function (path) { const mkdir = function (_path) { if (_path.indexOf(picturesPath) == 0) { - _path = _path.substring(picturesPath.length + 1); + _path = _path.substring(picturesPath.length); } let parts = _path.split("/"), path; + + console.log(parts, _path); + parts.unshift(picturesPath); return Promise.mapSeries(parts, function (part) { if (!path) { - path = picturesPath; + path = picturesPath.replace(/\/$/, ""); } else { path += "/" + part; } @@ -217,12 +104,12 @@ const exists = function(path) { } function convertRawToJpg(path, file) { - console.log("Converting " + path + "/" + file); + console.log("Converting " + path + file); path = picturesPath + path; return new Promise(function(resolve, reject) { - return exists(path + "/" + file.replace(rawExtension, ".jpg")).then(function(exist) { + return exists(path + file.replace(rawExtension, ".jpg")).then(function(exist) { if (exist) { console.log("Skipping already converted file: " + file); return; @@ -236,8 +123,8 @@ function convertRawToJpg(path, file) { "--compression=90", "--exif", "--overwrite", - "--output", path + "/" + file.replace(rawExtension, ".jpg"), - path + "/" + file + "--output", path + file.replace(rawExtension, ".jpg"), + path + file ]); const stderr = []; @@ -248,7 +135,7 @@ function convertRawToJpg(path, file) { return new Promise(function(ufraw, resolve, reject) { ufraw.on('exit', function(stderr, code, signal) { if (signal || code != 0) { - let error = "UFRAW for " + path + "/" + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n"; + let error = "UFRAW for " + path + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n"; console.error(error); return moveCorrupt(path, file).then(function() { console.warn("ufraw failed"); @@ -258,10 +145,10 @@ function convertRawToJpg(path, file) { return reject(error); }); } - return mkdir(path + "/raw").then(function() { - fs.rename(path + "/" + file, path + "/raw/" + file, function(err) { + return mkdir(path + "raw").then(function() { + fs.rename(path + file, path + "raw/" + file, function(err) { if (err) { - console.error("Unable to move RAW file: " + path + "/" + file); + console.error("Unable to move RAW file: " + path + file); return reject(err); } return resolve(); @@ -281,13 +168,13 @@ function moveCorrupt(path, file) { path = picturesPath + path; } - console.warn("Moving corrupt file '" + file + "' to " + path + "/corrupt"); + console.warn("Moving corrupt file '" + file + "' to " + path + "corrupt"); - return mkdir(path + "/corrupt").then(function() { + return mkdir(path + "corrupt").then(function() { return new Promise(function(resolve, reject) { - fs.rename(path + "/" + file, path + "/corrupt/" + file, function(err) { + fs.rename(path + file, path + "corrupt/" + file, function(err) { if (err) { - console.error("Unable to move corrupt file: " + path + "/" + file); + console.error("Unable to move corrupt file: " + path + file); return reject(err); } return resolve(); @@ -296,216 +183,40 @@ function moveCorrupt(path, file) { }); } -function triggerWatcher() { - setTimeout(triggerWatcher, 1000); - - if (!processRunning && processQueue.length) { - let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0); - processRunning = true; - - /* Sort to newest files to be processed first */ - processing.sort(function(a, b) { - return b[2] - a[2]; - }); - - return Promise.map(processing, function(entry) { - var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3]; - - let tmp = Promise.resolve(file); - /* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */ - if (rawExtension.exec(file)) { - tmp = exists(picturesPath + path + "/" + file.replace(rawExtension, ".jpg")).then(function(exist) { - if (exist) { - return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */ - } - - return mkdir(picturesPath + path + "/raw").then(function() { - return convertRawToJpg(path, file); - }).then(function() { - return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */ - }); - }); - } - - return tmp.then(function(file) { - var src = picturesPath + path + "/" + file, - image = sharp(src); - - return image.limitInputPixels(1073741824).metadata().then(function(metadata) { - if (metadata.exif) { - metadata.exif = exif(metadata.exif); - delete metadata.exif.thumbnail; - delete metadata.exif.image; - for (var key in metadata.exif.exif) { - if (Buffer.isBuffer(metadata.exif.exif[key])) { - metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]"; - } - } - } - - let replacements = { - albumId: albumId, - name: file.replace(/.[^.]*$/, ""), - path: path, - filename: file, - width: metadata.width, - height: metadata.height, - added: moment().format() - }; - - /* Ensure that top level images are placed into an album with a root path */ - replacements.path = replacements.path || "/"; - - if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) { - replacements.taken = moment(metadata.exif.exif.DateTimeOriginal).format(); - replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format(); - - if (replacements.taken == "Invalid date" || replacements.taken.replace(/T.*/, "") == "1899-11-30") { - console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif)); - replacements.taken = replacements.modified = moment(created).format(); - } - } else { - /* Attempt to infer the datestamp from the filename */ - let date = moment(created).format(); - - let match = file.match(/WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/); - if (match) { - date = moment((match[1]+" "+match[2]), "YYYY-MM-DD h.mm.ss a").format(); - if (date == "Invalid date") { - date = moment(created).format(); - } - } else { - match = file.match(/(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/); - if (match) { - if (match[2]) { /* Stamp had time in it */ - date = moment((match[1]+""+match[2]).replace(/-/g, ""), "YYYYMMDDHHmmss").format(); - } else { - date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format(); - } - if (date == "Invalid date") { - date = moment(created).format(); - } - } else { - date = moment(created).format(); - } - } - replacements.taken = replacements.modified = date; - } - - let dst = picturesPath + path + "/thumbs/" + file; - - return exists(dst).then(function(exist) { - if (exist) { - return; - } - - return image.resize(256, 256).toFile(dst).catch(function(error) { - console.error("Error resizing image: " + dst, error); - throw error; - }); - }).then(function() { - let dst = picturesPath + path + "/thumbs/scaled/" + file; - return exists(dst).then(function(exist) { - if (exist) { - return; - } - - return image.resize(Math.min(1024, metadata.width)).toFile(dst).catch(function(error) { - console.error("Error resizing image: " + dst, error); - throw error; - }); - }); - }).then(function() { - return photoDB.sequelize.query("INSERT INTO photos " + - "(albumId,path,filename,added,modified,taken,width,height,name)" + - "VALUES(:albumId,:path,:filename,DATETIME(:added),DATETIME(:modified),DATETIME(:taken),:width,:height,:name)", { - replacements: replacements - }); - }).then(function() { - toProcess--; - if (moment().add(-5, 'seconds') > lastMessage) { - console.log("Items to be processed: " + toProcess); - lastMessage = moment(); - } - }); - }).catch(function(error) { - console.error("Error reading image " + src + ": ", error); - return moveCorrupt(path, file).then(function() { - - /* If the original file was not a NEF/ORF, we are done... */ - if (!rawExtension.exec(entry[1])) { - return; - } - - /* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */ - for (var i = 0; i < triedClean.length; i++) { - if (triedClean[i] == path + "/" + file) { - /* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */ - // return moveCorrupt(path, entry[1]); - - console.error("Already attempted to convert NEF/ORF to JPG: " + path + "/" + file); - return; - } - } - - console.warn("Adding " + path + "/" + file + " back onto processing queue."); - triedClean.push(path + "/" + file); - processQueue.push([ path, file, created, albumId ]); - }); - }); - }).catch(function() { - console.warn("Continuing file processing."); - }); - }, { - concurrency: 1 - }).then(function() { - console.log("Completed processing queue."); - }); - } -} /*******************************************************************************************************/ -let processTimeout = null; function processBlock(items) { - if (!processTimeout) { - processTimeout = setTimeout(processBlock, 1000); - } - processTimeout = null; + /* Invoke once per second to check if there are items to process */ + setTimeout(processBlock, 1000); if (items) { processQueue = processQueue.concat(items); } - return; - /* 'needsProcessing' should only have their scanned stamp updated once thumbnails - * have been created. */ - console.log(needsProcessing.length + " had their HASH updated. Updating scanned stamp."); - return db.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:scanned)", { - replacements: { - scanned: needsProcessing - } - }); - if (!processRunning && processQueue.length) { let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0); processRunning = true; -console.log(processQueue.length); /* Sort to newest files to be processed first */ processing.sort(function(a, b) { - return b[2] - a[2]; + return a.stats.mtime - b.stats.mtime; }); - return Promise.map(processing, function(entry) { - var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3]; + return Promise.map(processing, function(asset) { + var path = asset.album.path, + file = asset.filename, + created = asset.stats.ctime, + albumId = asset.album.id; + console.log(picturesPath, path, file); + let tmp = Promise.resolve(file); /* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */ if (rawExtension.exec(file)) { - tmp = exists(picturesPath + path + "/" + file.replace(rawExtension, ".jpg")).then(function(exist) { + tmp = exists(picturesPath + path + file.replace(rawExtension, ".jpg")).then(function(exist) { if (exist) { return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */ } - return mkdir(picturesPath + path + "/raw").then(function() { + return mkdir(picturesPath + path + "raw").then(function() { return convertRawToJpg(path, file); }).then(function() { return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */ @@ -514,7 +225,7 @@ console.log(processQueue.length); } return tmp.then(function(file) { - var src = picturesPath + path + "/" + file, + var src = picturesPath + path + file, image = sharp(src); return image.limitInputPixels(1073741824).metadata().then(function(metadata) { @@ -529,26 +240,17 @@ console.log(processQueue.length); } } - let replacements = { - albumId: albumId, - name: file.replace(/.[^.]*$/, ""), - path: path, - filename: file, - width: metadata.width, - height: metadata.height, - added: moment().format() - }; - - /* Ensure that top level images are placed into an album with a root path */ - replacements.path = replacements.path || "/"; + asset.width = metadata.width; + asset.height = metadata.height; + asset.added = moment().format(); if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) { - replacements.taken = moment(metadata.exif.exif.DateTimeOriginal).format(); - replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format(); + asset.taken = moment(metadata.exif.exif.DateTimeOriginal).format(); + asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format(); - if (replacements.taken == "Invalid date" || replacements.taken.replace(/T.*/, "") == "1899-11-30") { + if (asset.taken == "Invalid date" || asset.taken.replace(/T.*/, "") == "1899-11-30") { console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif)); - replacements.taken = replacements.modified = moment(created).format(); + asset.taken = asset.modified = moment(created).format(); } } else { /* Attempt to infer the datestamp from the filename */ @@ -575,10 +277,10 @@ console.log(processQueue.length); date = moment(created).format(); } } - replacements.taken = replacements.modified = date; + asset.taken = asset.modified = date; } - let dst = picturesPath + path + "/thumbs/" + file; + let dst = picturesPath + path + "thumbs/" + file; return exists(dst).then(function(exist) { if (exist) { @@ -590,7 +292,7 @@ console.log(processQueue.length); throw error; }); }).then(function() { - let dst = picturesPath + path + "/thumbs/scaled/" + file; + let dst = picturesPath + path + "thumbs/scaled/" + file; return exists(dst).then(function(exist) { if (exist) { return; @@ -602,10 +304,10 @@ console.log(processQueue.length); }); }); }).then(function() { - return photoDB.sequelize.query("INSERT INTO photos " + - "(albumId,path,filename,added,modified,taken,width,height,name)" + - "VALUES(:albumId,:path,:filename,DATETIME(:added),DATETIME(:modified),DATETIME(:taken),:width,:height,:name)", { - replacements: replacements + return photoDB.sequelize.query("UPDATE photos SET " + + "(added,modified,taken,width,height,scanned)" + + "VALUES(:added,:modified,:taken,:width,:height,CURRENT_TIMESTAMP)", { + replacements: asset }); }).then(function() { toProcess--; @@ -619,23 +321,23 @@ console.log(processQueue.length); return moveCorrupt(path, file).then(function() { /* If the original file was not a NEF/ORF, we are done... */ - if (!rawExtension.exec(entry[1])) { + if (!rawExtension.exec(asset.filename)) { return; } /* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */ for (var i = 0; i < triedClean.length; i++) { - if (triedClean[i] == path + "/" + file) { + if (triedClean[i] == path + file) { /* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */ - // return moveCorrupt(path, entry[1]); + // return moveCorrupt(path, asset.filename); - console.error("Already attempted to convert NEF/ORF to JPG: " + path + "/" + file); + console.error("Already attempted to convert NEF/ORF to JPG: " + path + file); return; } } - console.warn("Adding " + path + "/" + file + " back onto processing queue."); - triedClean.push(path + "/" + file); + console.warn("Adding " + path + file + " back onto processing queue."); + triedClean.push(path + file); processQueue.push([ path, file, created, albumId ]); }); }); @@ -654,7 +356,7 @@ console.log(processQueue.length); function scanDir(parent, path) { let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"), album = { - path: path.slice(picturesPath.length) + "/", + path: path.slice(picturesPath.length), /* path already ends in '/' */ name: path.replace(/.*\//, "").replace(/_/g, " "), parent: parent, allAssetCount: 0, @@ -692,10 +394,10 @@ function scanDir(parent, path) { }); }).then(function(files) { return Promise.map(files, function(file) { - let filepath = path + "/" + file; - + let filepath = path + file; return stat(filepath).then(function(stats) { if (stats.isDirectory()) { + filepath += "/"; return scanDir(album, filepath).spread(function(_albums, _assets) { album.allAssetCount += _assets.length; album.allAlbumCount += _albums.length + 1; @@ -710,7 +412,9 @@ function scanDir(parent, path) { if (!re.exec(file)) { return; } - + + album.hasAssets = true; + assets.push({ filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */ name: file.replace(/.[^.]*$/, ""), @@ -719,6 +423,12 @@ function scanDir(parent, path) { }); }); }); + }).then(function() { + return Promise.map(albums, function(album) { + if (album.hasAssets) { + return mkdir(album.path + "thumbs"); + } + }); }).then(function() { return [ albums, assets ]; }); @@ -904,7 +614,7 @@ module.exports = { replacements: asset }).then(function() { /* HASH has been updated; add to the needsProcessing array */ - needsProcessing.push(asset.id); + needsProcessing.push(asset); }); }); }).then(function() {