Switch to using transactions

Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2018-09-28 11:54:16 -07:00
parent 1566d132f6
commit 8b89ce7507
2 changed files with 63 additions and 56 deletions

View File

@ -132,7 +132,6 @@ router.get("/*", function(req, res/*, next*/) {
photos.forEach(function(photo) { photos.forEach(function(photo) {
for (var key in photo) { for (var key in photo) {
if (photo[key] instanceof Date) { if (photo[key] instanceof Date) {
photo[key].setHours(0, 0, 0, 0);
photo[key] = moment(photo[key]); photo[key] = moment(photo[key]);
} }
} }

View File

@ -217,6 +217,7 @@ function processBlock(items) {
}).then(function() { }).then(function() {
let toProcess = processing.length, lastMessage = moment(); let toProcess = processing.length, lastMessage = moment();
/* Needs to be one at a time in case there are multiple HASH collisions */ /* Needs to be one at a time in case there are multiple HASH collisions */
return photoDB.sequelize.transaction(function(transaction) {
return Promise.mapSeries(processing, function(asset) { return Promise.mapSeries(processing, function(asset) {
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " + return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " + "LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
@ -246,7 +247,8 @@ function processBlock(items) {
} }
return photoDB.sequelize.query(query, { return photoDB.sequelize.query(query, {
replacements: asset replacements: asset,
transaction: transaction
}); });
}).then(function() { }).then(function() {
toProcess--; toProcess--;
@ -256,13 +258,14 @@ function processBlock(items) {
} }
}); });
}); });
});
}).then(function() { }).then(function() {
let toProcess = processing.length, lastMessage = moment(); let toProcess = processing.length, lastMessage = moment();
console.log(needsProcessing.length + " assets need to have metadata extracted"); console.log(needsProcessing.length + " assets need to have metadata extracted");
return Promise.map(needsProcessing, function(asset) { return Promise.map(needsProcessing, function(asset) {
var path = asset.album.path, var path = asset.album.path,
file = asset.filename, file = asset.filename,
created = asset.stats.ctime, created = asset.stats.mtime,
albumId = asset.album.id; albumId = asset.album.id;
let tmp = Promise.resolve(file); let tmp = Promise.resolve(file);
@ -364,7 +367,7 @@ function processBlock(items) {
return photoDB.sequelize.query("UPDATE photos SET " + return photoDB.sequelize.query("UPDATE photos SET " +
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP " + "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP " +
"WHERE id=:id", { "WHERE id=:id", {
replacements: asset replacements: asset,
}); });
}).then(function() { }).then(function() {
toProcess--; toProcess--;
@ -530,7 +533,6 @@ function scanDir(parent, path) {
filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */ filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
name: file.replace(/.[^.]*$/, ""), name: file.replace(/.[^.]*$/, ""),
stats: { stats: {
ctime: stats.ctime,
mtime: stats.mtime mtime: stats.mtime
}, },
album: album album: album
@ -548,7 +550,7 @@ function scanDir(parent, path) {
}); });
} }
function findOrCreateDBAlbum(album) { function findOrCreateDBAlbum(transaction, album) {
let query = "SELECT id FROM albums WHERE path=:path AND "; let query = "SELECT id FROM albums WHERE path=:path AND ";
if (!album.parent) { if (!album.parent) {
query += "parentId IS NULL"; query += "parentId IS NULL";
@ -572,7 +574,8 @@ function findOrCreateDBAlbum(album) {
console.warn("Creating top level album: " + picturesPath); console.warn("Creating top level album: " + picturesPath);
} }
return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", { return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
replacements: album replacements: album,
transaction: transaction
}).spread(function(results, metadata) { }).spread(function(results, metadata) {
return metadata.lastID; return metadata.lastID;
}); });
@ -585,8 +588,8 @@ function findOrCreateDBAlbum(album) {
}); });
} }
function findOrUpdateDBAsset(asset) { function findOrUpdateDBAsset(transaction, asset) {
let query = "SELECT id,scanned FROM photos WHERE albumId=:albumId AND filename=:filename"; let query = "SELECT id FROM photos WHERE albumId=:albumId AND filename=:filename";
if (!asset.album || !asset.album.id) { if (!asset.album || !asset.album.id) {
let error = "Asset being processed without an album"; let error = "Asset being processed without an album";
console.error(error); console.error(error);
@ -601,7 +604,8 @@ function findOrUpdateDBAsset(asset) {
return photoDB.sequelize.query("INSERT INTO photos " + return photoDB.sequelize.query("INSERT INTO photos " +
"(albumId,filename,name) " + "(albumId,filename,name) " +
"VALUES(:albumId,:filename,:name)", { "VALUES(:albumId,:filename,:name)", {
replacements: asset replacements: asset,
transaction: transaction
}).spread(function(results, metadata) { }).spread(function(results, metadata) {
return [ metadata.lastID, null ]; return [ metadata.lastID, null ];
}); });
@ -668,27 +672,31 @@ module.exports = {
now = Date.now(); now = Date.now();
let toProcess = albums.length, lastMessage = moment(); let toProcess = albums.length, lastMessage = moment();
return photoDB.sequelize.transaction(function(transaction) {
return Promise.mapSeries(albums, function(album) { return Promise.mapSeries(albums, function(album) {
return findOrCreateDBAlbum(album).then(function() { return findOrCreateDBAlbum(transaction, album).then(function() {
toProcess--; toProcess--;
if (moment().add(-5, 'seconds') > lastMessage) { if (moment().add(-5, 'seconds') > lastMessage) {
console.log("Albums to be created in DB: " + toProcess); console.log("Albums to be created in DB: " + toProcess);
lastMessage = moment(); lastMessage = moment();
} }
}); });
});
}).then(function() { }).then(function() {
console.log("Processed " + albums.length + " album DB entries in " + console.log("Processed " + albums.length + " album DB entries in " +
((Date.now() - now) / 1000) + "s"); ((Date.now() - now) / 1000) + "s");
now = Date.now(); now = Date.now();
let processed = 0, start = Date.now(), last = 0, updateScanned = []; let processed = 0, start = Date.now(), last = 0, updateScanned = [];
return photoDB.sequelize.transaction(function(transaction) {
return Promise.map(assets, function(asset) { return Promise.map(assets, function(asset) {
return findOrUpdateDBAsset(asset).then(function(asset) { return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
if (asset.scanned < asset.stats.mtime) { if (asset.scanned < asset.stats.mtime) {
needsProcessing.push(asset); needsProcessing.push(asset);
} else { } else {
updateScanned.push(asset.id); updateScanned.push(asset.id);
} }
});
}).then(function(asset) { }).then(function(asset) {
processed++; processed++;