From 8b89ce75077b10de89d9ce6e13522dafb9d5af22 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Fri, 28 Sep 2018 11:54:16 -0700 Subject: [PATCH] Switch to using transactions Signed-off-by: James Ketrenos --- server/routes/photos.js | 1 - server/scanner.js | 118 +++++++++++++++++++++------------------- 2 files changed, 63 insertions(+), 56 deletions(-) diff --git a/server/routes/photos.js b/server/routes/photos.js index 8619861..5575d89 100755 --- a/server/routes/photos.js +++ b/server/routes/photos.js @@ -132,7 +132,6 @@ router.get("/*", function(req, res/*, next*/) { photos.forEach(function(photo) { for (var key in photo) { if (photo[key] instanceof Date) { - photo[key].setHours(0, 0, 0, 0); photo[key] = moment(photo[key]); } } diff --git a/server/scanner.js b/server/scanner.js index 8d39d6a..c6e26a6 100755 --- a/server/scanner.js +++ b/server/scanner.js @@ -217,43 +217,46 @@ function processBlock(items) { }).then(function() { let toProcess = processing.length, lastMessage = moment(); /* Needs to be one at a time in case there are multiple HASH collisions */ - return Promise.mapSeries(processing, function(asset) { - return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " + - "LEFT JOIN photos ON (photos.id=photohashes.photoId) " + - "LEFT JOIN albums ON (albums.id=photos.albumId) " + - "WHERE hash=:hash OR photoId=:id", { - replacements: asset, - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(results) { - let query; - if (results.length == 0) { - query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; - } else if (results[0].hash != asset.hash) { - query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)"; - } else if (results[0].photoId != asset.id) { - console.log("Duplicate asset: " + - "'" + asset.album.path + asset.filename + "' is a copy of " + - "'" + results[0].path + results[0].filename + "'"); - duplicates.push(asset); - return; - } + return photoDB.sequelize.transaction(function(transaction) { + return Promise.mapSeries(processing, function(asset) { + return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " + + "LEFT JOIN photos ON (photos.id=photohashes.photoId) " + + "LEFT JOIN albums ON (albums.id=photos.albumId) " + + "WHERE hash=:hash OR photoId=:id", { + replacements: asset, + type: photoDB.sequelize.QueryTypes.SELECT + }).then(function(results) { + let query; + if (results.length == 0) { + query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; + } else if (results[0].hash != asset.hash) { + query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)"; + } else if (results[0].photoId != asset.id) { + console.log("Duplicate asset: " + + "'" + asset.album.path + asset.filename + "' is a copy of " + + "'" + results[0].path + results[0].filename + "'"); + duplicates.push(asset); + return; + } - /* Even if the hash doesn't need to be updated, the entry needs to be scanned */ - needsProcessing.push(asset); + /* Even if the hash doesn't need to be updated, the entry needs to be scanned */ + needsProcessing.push(asset); - if (!query) { - return; - } + if (!query) { + return; + } - return photoDB.sequelize.query(query, { - replacements: asset + return photoDB.sequelize.query(query, { + replacements: asset, + transaction: transaction + }); + }).then(function() { + toProcess--; + if (moment().add(-5, 'seconds') > lastMessage) { + console.log("Hash items to be checked: " + toProcess); + lastMessage = moment(); + } }); - }).then(function() { - toProcess--; - if (moment().add(-5, 'seconds') > lastMessage) { - console.log("Hash items to be checked: " + toProcess); - lastMessage = moment(); - } }); }); }).then(function() { @@ -262,7 +265,7 @@ function processBlock(items) { return Promise.map(needsProcessing, function(asset) { var path = asset.album.path, file = asset.filename, - created = asset.stats.ctime, + created = asset.stats.mtime, albumId = asset.album.id; let tmp = Promise.resolve(file); @@ -364,7 +367,7 @@ function processBlock(items) { return photoDB.sequelize.query("UPDATE photos SET " + "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP " + "WHERE id=:id", { - replacements: asset + replacements: asset, }); }).then(function() { toProcess--; @@ -530,7 +533,6 @@ function scanDir(parent, path) { filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */ name: file.replace(/.[^.]*$/, ""), stats: { - ctime: stats.ctime, mtime: stats.mtime }, album: album @@ -548,7 +550,7 @@ function scanDir(parent, path) { }); } -function findOrCreateDBAlbum(album) { +function findOrCreateDBAlbum(transaction, album) { let query = "SELECT id FROM albums WHERE path=:path AND "; if (!album.parent) { query += "parentId IS NULL"; @@ -572,7 +574,8 @@ function findOrCreateDBAlbum(album) { console.warn("Creating top level album: " + picturesPath); } return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", { - replacements: album + replacements: album, + transaction: transaction }).spread(function(results, metadata) { return metadata.lastID; }); @@ -585,8 +588,8 @@ function findOrCreateDBAlbum(album) { }); } -function findOrUpdateDBAsset(asset) { - let query = "SELECT id,scanned FROM photos WHERE albumId=:albumId AND filename=:filename"; +function findOrUpdateDBAsset(transaction, asset) { + let query = "SELECT id FROM photos WHERE albumId=:albumId AND filename=:filename"; if (!asset.album || !asset.album.id) { let error = "Asset being processed without an album"; console.error(error); @@ -601,7 +604,8 @@ function findOrUpdateDBAsset(asset) { return photoDB.sequelize.query("INSERT INTO photos " + "(albumId,filename,name) " + "VALUES(:albumId,:filename,:name)", { - replacements: asset + replacements: asset, + transaction: transaction }).spread(function(results, metadata) { return [ metadata.lastID, null ]; }); @@ -668,13 +672,15 @@ module.exports = { now = Date.now(); let toProcess = albums.length, lastMessage = moment(); - return Promise.mapSeries(albums, function(album) { - return findOrCreateDBAlbum(album).then(function() { - toProcess--; - if (moment().add(-5, 'seconds') > lastMessage) { - console.log("Albums to be created in DB: " + toProcess); - lastMessage = moment(); - } + return photoDB.sequelize.transaction(function(transaction) { + return Promise.mapSeries(albums, function(album) { + return findOrCreateDBAlbum(transaction, album).then(function() { + toProcess--; + if (moment().add(-5, 'seconds') > lastMessage) { + console.log("Albums to be created in DB: " + toProcess); + lastMessage = moment(); + } + }); }); }).then(function() { console.log("Processed " + albums.length + " album DB entries in " + @@ -682,13 +688,15 @@ module.exports = { now = Date.now(); let processed = 0, start = Date.now(), last = 0, updateScanned = []; - return Promise.map(assets, function(asset) { - return findOrUpdateDBAsset(asset).then(function(asset) { - if (asset.scanned < asset.stats.mtime) { - needsProcessing.push(asset); - } else { - updateScanned.push(asset.id); - } + return photoDB.sequelize.transaction(function(transaction) { + return Promise.map(assets, function(asset) { + return findOrUpdateDBAsset(transaction, asset).then(function(asset) { + if (asset.scanned < asset.stats.mtime) { + needsProcessing.push(asset); + } else { + updateScanned.push(asset.id); + } + }); }).then(function(asset) { processed++;