Switch to using transactions
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
parent
1566d132f6
commit
8b89ce7507
@ -132,7 +132,6 @@ router.get("/*", function(req, res/*, next*/) {
|
||||
photos.forEach(function(photo) {
|
||||
for (var key in photo) {
|
||||
if (photo[key] instanceof Date) {
|
||||
photo[key].setHours(0, 0, 0, 0);
|
||||
photo[key] = moment(photo[key]);
|
||||
}
|
||||
}
|
||||
|
||||
@ -217,43 +217,46 @@ function processBlock(items) {
|
||||
}).then(function() {
|
||||
let toProcess = processing.length, lastMessage = moment();
|
||||
/* Needs to be one at a time in case there are multiple HASH collisions */
|
||||
return Promise.mapSeries(processing, function(asset) {
|
||||
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
|
||||
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
|
||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
||||
"WHERE hash=:hash OR photoId=:id", {
|
||||
replacements: asset,
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}).then(function(results) {
|
||||
let query;
|
||||
if (results.length == 0) {
|
||||
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
|
||||
} else if (results[0].hash != asset.hash) {
|
||||
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
|
||||
} else if (results[0].photoId != asset.id) {
|
||||
console.log("Duplicate asset: " +
|
||||
"'" + asset.album.path + asset.filename + "' is a copy of " +
|
||||
"'" + results[0].path + results[0].filename + "'");
|
||||
duplicates.push(asset);
|
||||
return;
|
||||
}
|
||||
return photoDB.sequelize.transaction(function(transaction) {
|
||||
return Promise.mapSeries(processing, function(asset) {
|
||||
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
|
||||
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
|
||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
||||
"WHERE hash=:hash OR photoId=:id", {
|
||||
replacements: asset,
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}).then(function(results) {
|
||||
let query;
|
||||
if (results.length == 0) {
|
||||
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
|
||||
} else if (results[0].hash != asset.hash) {
|
||||
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
|
||||
} else if (results[0].photoId != asset.id) {
|
||||
console.log("Duplicate asset: " +
|
||||
"'" + asset.album.path + asset.filename + "' is a copy of " +
|
||||
"'" + results[0].path + results[0].filename + "'");
|
||||
duplicates.push(asset);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
|
||||
needsProcessing.push(asset);
|
||||
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
|
||||
needsProcessing.push(asset);
|
||||
|
||||
if (!query) {
|
||||
return;
|
||||
}
|
||||
if (!query) {
|
||||
return;
|
||||
}
|
||||
|
||||
return photoDB.sequelize.query(query, {
|
||||
replacements: asset
|
||||
return photoDB.sequelize.query(query, {
|
||||
replacements: asset,
|
||||
transaction: transaction
|
||||
});
|
||||
}).then(function() {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
console.log("Hash items to be checked: " + toProcess);
|
||||
lastMessage = moment();
|
||||
}
|
||||
});
|
||||
}).then(function() {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
console.log("Hash items to be checked: " + toProcess);
|
||||
lastMessage = moment();
|
||||
}
|
||||
});
|
||||
});
|
||||
}).then(function() {
|
||||
@ -262,7 +265,7 @@ function processBlock(items) {
|
||||
return Promise.map(needsProcessing, function(asset) {
|
||||
var path = asset.album.path,
|
||||
file = asset.filename,
|
||||
created = asset.stats.ctime,
|
||||
created = asset.stats.mtime,
|
||||
albumId = asset.album.id;
|
||||
|
||||
let tmp = Promise.resolve(file);
|
||||
@ -364,7 +367,7 @@ function processBlock(items) {
|
||||
return photoDB.sequelize.query("UPDATE photos SET " +
|
||||
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP " +
|
||||
"WHERE id=:id", {
|
||||
replacements: asset
|
||||
replacements: asset,
|
||||
});
|
||||
}).then(function() {
|
||||
toProcess--;
|
||||
@ -530,7 +533,6 @@ function scanDir(parent, path) {
|
||||
filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
|
||||
name: file.replace(/.[^.]*$/, ""),
|
||||
stats: {
|
||||
ctime: stats.ctime,
|
||||
mtime: stats.mtime
|
||||
},
|
||||
album: album
|
||||
@ -548,7 +550,7 @@ function scanDir(parent, path) {
|
||||
});
|
||||
}
|
||||
|
||||
function findOrCreateDBAlbum(album) {
|
||||
function findOrCreateDBAlbum(transaction, album) {
|
||||
let query = "SELECT id FROM albums WHERE path=:path AND ";
|
||||
if (!album.parent) {
|
||||
query += "parentId IS NULL";
|
||||
@ -572,7 +574,8 @@ function findOrCreateDBAlbum(album) {
|
||||
console.warn("Creating top level album: " + picturesPath);
|
||||
}
|
||||
return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
|
||||
replacements: album
|
||||
replacements: album,
|
||||
transaction: transaction
|
||||
}).spread(function(results, metadata) {
|
||||
return metadata.lastID;
|
||||
});
|
||||
@ -585,8 +588,8 @@ function findOrCreateDBAlbum(album) {
|
||||
});
|
||||
}
|
||||
|
||||
function findOrUpdateDBAsset(asset) {
|
||||
let query = "SELECT id,scanned FROM photos WHERE albumId=:albumId AND filename=:filename";
|
||||
function findOrUpdateDBAsset(transaction, asset) {
|
||||
let query = "SELECT id FROM photos WHERE albumId=:albumId AND filename=:filename";
|
||||
if (!asset.album || !asset.album.id) {
|
||||
let error = "Asset being processed without an album";
|
||||
console.error(error);
|
||||
@ -601,7 +604,8 @@ function findOrUpdateDBAsset(asset) {
|
||||
return photoDB.sequelize.query("INSERT INTO photos " +
|
||||
"(albumId,filename,name) " +
|
||||
"VALUES(:albumId,:filename,:name)", {
|
||||
replacements: asset
|
||||
replacements: asset,
|
||||
transaction: transaction
|
||||
}).spread(function(results, metadata) {
|
||||
return [ metadata.lastID, null ];
|
||||
});
|
||||
@ -668,13 +672,15 @@ module.exports = {
|
||||
now = Date.now();
|
||||
|
||||
let toProcess = albums.length, lastMessage = moment();
|
||||
return Promise.mapSeries(albums, function(album) {
|
||||
return findOrCreateDBAlbum(album).then(function() {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
console.log("Albums to be created in DB: " + toProcess);
|
||||
lastMessage = moment();
|
||||
}
|
||||
return photoDB.sequelize.transaction(function(transaction) {
|
||||
return Promise.mapSeries(albums, function(album) {
|
||||
return findOrCreateDBAlbum(transaction, album).then(function() {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
console.log("Albums to be created in DB: " + toProcess);
|
||||
lastMessage = moment();
|
||||
}
|
||||
});
|
||||
});
|
||||
}).then(function() {
|
||||
console.log("Processed " + albums.length + " album DB entries in " +
|
||||
@ -682,13 +688,15 @@ module.exports = {
|
||||
now = Date.now();
|
||||
|
||||
let processed = 0, start = Date.now(), last = 0, updateScanned = [];
|
||||
return Promise.map(assets, function(asset) {
|
||||
return findOrUpdateDBAsset(asset).then(function(asset) {
|
||||
if (asset.scanned < asset.stats.mtime) {
|
||||
needsProcessing.push(asset);
|
||||
} else {
|
||||
updateScanned.push(asset.id);
|
||||
}
|
||||
return photoDB.sequelize.transaction(function(transaction) {
|
||||
return Promise.map(assets, function(asset) {
|
||||
return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
|
||||
if (asset.scanned < asset.stats.mtime) {
|
||||
needsProcessing.push(asset);
|
||||
} else {
|
||||
updateScanned.push(asset.id);
|
||||
}
|
||||
});
|
||||
}).then(function(asset) {
|
||||
processed++;
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user