jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Switch to using transactions

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2018-09-28 11:54:16 -07:00
parent 1566d132f6
commit 8b89ce7507
2 changed files with 63 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/routes/photos.js
View File

@ -132,7 +132,6 @@ router.get("/*", function(req, res/*, next*/) {
photos.forEach(function(photo) { photos.forEach(function(photo) {
for (var key in photo) { for (var key in photo) {
if (photo[key] instanceof Date) { if (photo[key] instanceof Date) {
photo[key].setHours(0, 0, 0, 0);
photo[key] = moment(photo[key]); photo[key] = moment(photo[key]);
} }
} }

118
server/scanner.js
View File

@ -217,43 +217,46 @@ function processBlock(items) {
}).then(function() { }).then(function() {
let toProcess = processing.length, lastMessage = moment(); let toProcess = processing.length, lastMessage = moment();
/* Needs to be one at a time in case there are multiple HASH collisions */ /* Needs to be one at a time in case there are multiple HASH collisions */
return Promise.mapSeries(processing, function(asset) { return photoDB.sequelize.transaction(function(transaction) {
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " + return Promise.mapSeries(processing, function(asset) {
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " + return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
"LEFT JOIN albums ON (albums.id=photos.albumId) " + "LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
"WHERE hash=:hash OR photoId=:id", { "LEFT JOIN albums ON (albums.id=photos.albumId) " +
replacements: asset, "WHERE hash=:hash OR photoId=:id", {
type: photoDB.sequelize.QueryTypes.SELECT replacements: asset,
}).then(function(results) { type: photoDB.sequelize.QueryTypes.SELECT
let query; }).then(function(results) {
if (results.length == 0) { let query;
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; if (results.length == 0) {
} else if (results[0].hash != asset.hash) { query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)"; } else if (results[0].hash != asset.hash) {
} else if (results[0].photoId != asset.id) { query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
console.log("Duplicate asset: " + } else if (results[0].photoId != asset.id) {
"'" + asset.album.path + asset.filename + "' is a copy of " + console.log("Duplicate asset: " +
"'" + results[0].path + results[0].filename + "'"); "'" + asset.album.path + asset.filename + "' is a copy of " +
duplicates.push(asset); "'" + results[0].path + results[0].filename + "'");
return; duplicates.push(asset);
} return;
}
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */ /* Even if the hash doesn't need to be updated, the entry needs to be scanned */
needsProcessing.push(asset); needsProcessing.push(asset);
if (!query) { if (!query) {
return; return;
} }
return photoDB.sequelize.query(query, { return photoDB.sequelize.query(query, {
replacements: asset replacements: asset,
transaction: transaction
});
}).then(function() {
toProcess--;
if (moment().add(-5, 'seconds') > lastMessage) {
console.log("Hash items to be checked: " + toProcess);
lastMessage = moment();
}
}); });
}).then(function() {
toProcess--;
if (moment().add(-5, 'seconds') > lastMessage) {
console.log("Hash items to be checked: " + toProcess);
lastMessage = moment();
}
}); });
}); });
}).then(function() { }).then(function() {
@ -262,7 +265,7 @@ function processBlock(items) {
return Promise.map(needsProcessing, function(asset) { return Promise.map(needsProcessing, function(asset) {
var path = asset.album.path, var path = asset.album.path,
file = asset.filename, file = asset.filename,
created = asset.stats.ctime, created = asset.stats.mtime,
albumId = asset.album.id; albumId = asset.album.id;
let tmp = Promise.resolve(file); let tmp = Promise.resolve(file);
@ -364,7 +367,7 @@ function processBlock(items) {
return photoDB.sequelize.query("UPDATE photos SET " + return photoDB.sequelize.query("UPDATE photos SET " +
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP " + "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP " +
"WHERE id=:id", { "WHERE id=:id", {
replacements: asset replacements: asset,
}); });
}).then(function() { }).then(function() {
toProcess--; toProcess--;
@ -530,7 +533,6 @@ function scanDir(parent, path) {
filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */ filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
name: file.replace(/.[^.]*$/, ""), name: file.replace(/.[^.]*$/, ""),
stats: { stats: {
ctime: stats.ctime,
mtime: stats.mtime mtime: stats.mtime
}, },
album: album album: album
@ -548,7 +550,7 @@ function scanDir(parent, path) {
}); });
} }
function findOrCreateDBAlbum(album) { function findOrCreateDBAlbum(transaction, album) {
let query = "SELECT id FROM albums WHERE path=:path AND "; let query = "SELECT id FROM albums WHERE path=:path AND ";
if (!album.parent) { if (!album.parent) {
query += "parentId IS NULL"; query += "parentId IS NULL";
@ -572,7 +574,8 @@ function findOrCreateDBAlbum(album) {
console.warn("Creating top level album: " + picturesPath); console.warn("Creating top level album: " + picturesPath);
} }
return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", { return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
replacements: album replacements: album,
transaction: transaction
}).spread(function(results, metadata) { }).spread(function(results, metadata) {
return metadata.lastID; return metadata.lastID;
}); });
@ -585,8 +588,8 @@ function findOrCreateDBAlbum(album) {
}); });
} }
function findOrUpdateDBAsset(asset) { function findOrUpdateDBAsset(transaction, asset) {
let query = "SELECT id,scanned FROM photos WHERE albumId=:albumId AND filename=:filename"; let query = "SELECT id FROM photos WHERE albumId=:albumId AND filename=:filename";
if (!asset.album || !asset.album.id) { if (!asset.album || !asset.album.id) {
let error = "Asset being processed without an album"; let error = "Asset being processed without an album";
console.error(error); console.error(error);
@ -601,7 +604,8 @@ function findOrUpdateDBAsset(asset) {
return photoDB.sequelize.query("INSERT INTO photos " + return photoDB.sequelize.query("INSERT INTO photos " +
"(albumId,filename,name) " + "(albumId,filename,name) " +
"VALUES(:albumId,:filename,:name)", { "VALUES(:albumId,:filename,:name)", {
replacements: asset replacements: asset,
transaction: transaction
}).spread(function(results, metadata) { }).spread(function(results, metadata) {
return [ metadata.lastID, null ]; return [ metadata.lastID, null ];
}); });
@ -668,13 +672,15 @@ module.exports = {
now = Date.now(); now = Date.now();
let toProcess = albums.length, lastMessage = moment(); let toProcess = albums.length, lastMessage = moment();
return Promise.mapSeries(albums, function(album) { return photoDB.sequelize.transaction(function(transaction) {
return findOrCreateDBAlbum(album).then(function() { return Promise.mapSeries(albums, function(album) {
toProcess--; return findOrCreateDBAlbum(transaction, album).then(function() {
if (moment().add(-5, 'seconds') > lastMessage) { toProcess--;
console.log("Albums to be created in DB: " + toProcess); if (moment().add(-5, 'seconds') > lastMessage) {
lastMessage = moment(); console.log("Albums to be created in DB: " + toProcess);
} lastMessage = moment();
}
});
}); });
}).then(function() { }).then(function() {
console.log("Processed " + albums.length + " album DB entries in " + console.log("Processed " + albums.length + " album DB entries in " +
@ -682,13 +688,15 @@ module.exports = {
now = Date.now(); now = Date.now();
let processed = 0, start = Date.now(), last = 0, updateScanned = []; let processed = 0, start = Date.now(), last = 0, updateScanned = [];
return Promise.map(assets, function(asset) { return photoDB.sequelize.transaction(function(transaction) {
return findOrUpdateDBAsset(asset).then(function(asset) { return Promise.map(assets, function(asset) {
if (asset.scanned < asset.stats.mtime) { return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
needsProcessing.push(asset); if (asset.scanned < asset.stats.mtime) {
} else { needsProcessing.push(asset);
updateScanned.push(asset.id); } else {
} updateScanned.push(asset.id);
}
});
}).then(function(asset) { }).then(function(asset) {
processed++; processed++;