Switch to using transactions
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
parent
1566d132f6
commit
8b89ce7507
@ -132,7 +132,6 @@ router.get("/*", function(req, res/*, next*/) {
|
|||||||
photos.forEach(function(photo) {
|
photos.forEach(function(photo) {
|
||||||
for (var key in photo) {
|
for (var key in photo) {
|
||||||
if (photo[key] instanceof Date) {
|
if (photo[key] instanceof Date) {
|
||||||
photo[key].setHours(0, 0, 0, 0);
|
|
||||||
photo[key] = moment(photo[key]);
|
photo[key] = moment(photo[key]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -217,6 +217,7 @@ function processBlock(items) {
|
|||||||
}).then(function() {
|
}).then(function() {
|
||||||
let toProcess = processing.length, lastMessage = moment();
|
let toProcess = processing.length, lastMessage = moment();
|
||||||
/* Needs to be one at a time in case there are multiple HASH collisions */
|
/* Needs to be one at a time in case there are multiple HASH collisions */
|
||||||
|
return photoDB.sequelize.transaction(function(transaction) {
|
||||||
return Promise.mapSeries(processing, function(asset) {
|
return Promise.mapSeries(processing, function(asset) {
|
||||||
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
|
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
|
||||||
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
|
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
|
||||||
@ -246,7 +247,8 @@ function processBlock(items) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return photoDB.sequelize.query(query, {
|
return photoDB.sequelize.query(query, {
|
||||||
replacements: asset
|
replacements: asset,
|
||||||
|
transaction: transaction
|
||||||
});
|
});
|
||||||
}).then(function() {
|
}).then(function() {
|
||||||
toProcess--;
|
toProcess--;
|
||||||
@ -256,13 +258,14 @@ function processBlock(items) {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
});
|
||||||
}).then(function() {
|
}).then(function() {
|
||||||
let toProcess = processing.length, lastMessage = moment();
|
let toProcess = processing.length, lastMessage = moment();
|
||||||
console.log(needsProcessing.length + " assets need to have metadata extracted");
|
console.log(needsProcessing.length + " assets need to have metadata extracted");
|
||||||
return Promise.map(needsProcessing, function(asset) {
|
return Promise.map(needsProcessing, function(asset) {
|
||||||
var path = asset.album.path,
|
var path = asset.album.path,
|
||||||
file = asset.filename,
|
file = asset.filename,
|
||||||
created = asset.stats.ctime,
|
created = asset.stats.mtime,
|
||||||
albumId = asset.album.id;
|
albumId = asset.album.id;
|
||||||
|
|
||||||
let tmp = Promise.resolve(file);
|
let tmp = Promise.resolve(file);
|
||||||
@ -364,7 +367,7 @@ function processBlock(items) {
|
|||||||
return photoDB.sequelize.query("UPDATE photos SET " +
|
return photoDB.sequelize.query("UPDATE photos SET " +
|
||||||
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP " +
|
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP " +
|
||||||
"WHERE id=:id", {
|
"WHERE id=:id", {
|
||||||
replacements: asset
|
replacements: asset,
|
||||||
});
|
});
|
||||||
}).then(function() {
|
}).then(function() {
|
||||||
toProcess--;
|
toProcess--;
|
||||||
@ -530,7 +533,6 @@ function scanDir(parent, path) {
|
|||||||
filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
|
filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
|
||||||
name: file.replace(/.[^.]*$/, ""),
|
name: file.replace(/.[^.]*$/, ""),
|
||||||
stats: {
|
stats: {
|
||||||
ctime: stats.ctime,
|
|
||||||
mtime: stats.mtime
|
mtime: stats.mtime
|
||||||
},
|
},
|
||||||
album: album
|
album: album
|
||||||
@ -548,7 +550,7 @@ function scanDir(parent, path) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function findOrCreateDBAlbum(album) {
|
function findOrCreateDBAlbum(transaction, album) {
|
||||||
let query = "SELECT id FROM albums WHERE path=:path AND ";
|
let query = "SELECT id FROM albums WHERE path=:path AND ";
|
||||||
if (!album.parent) {
|
if (!album.parent) {
|
||||||
query += "parentId IS NULL";
|
query += "parentId IS NULL";
|
||||||
@ -572,7 +574,8 @@ function findOrCreateDBAlbum(album) {
|
|||||||
console.warn("Creating top level album: " + picturesPath);
|
console.warn("Creating top level album: " + picturesPath);
|
||||||
}
|
}
|
||||||
return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
|
return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
|
||||||
replacements: album
|
replacements: album,
|
||||||
|
transaction: transaction
|
||||||
}).spread(function(results, metadata) {
|
}).spread(function(results, metadata) {
|
||||||
return metadata.lastID;
|
return metadata.lastID;
|
||||||
});
|
});
|
||||||
@ -585,8 +588,8 @@ function findOrCreateDBAlbum(album) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function findOrUpdateDBAsset(asset) {
|
function findOrUpdateDBAsset(transaction, asset) {
|
||||||
let query = "SELECT id,scanned FROM photos WHERE albumId=:albumId AND filename=:filename";
|
let query = "SELECT id FROM photos WHERE albumId=:albumId AND filename=:filename";
|
||||||
if (!asset.album || !asset.album.id) {
|
if (!asset.album || !asset.album.id) {
|
||||||
let error = "Asset being processed without an album";
|
let error = "Asset being processed without an album";
|
||||||
console.error(error);
|
console.error(error);
|
||||||
@ -601,7 +604,8 @@ function findOrUpdateDBAsset(asset) {
|
|||||||
return photoDB.sequelize.query("INSERT INTO photos " +
|
return photoDB.sequelize.query("INSERT INTO photos " +
|
||||||
"(albumId,filename,name) " +
|
"(albumId,filename,name) " +
|
||||||
"VALUES(:albumId,:filename,:name)", {
|
"VALUES(:albumId,:filename,:name)", {
|
||||||
replacements: asset
|
replacements: asset,
|
||||||
|
transaction: transaction
|
||||||
}).spread(function(results, metadata) {
|
}).spread(function(results, metadata) {
|
||||||
return [ metadata.lastID, null ];
|
return [ metadata.lastID, null ];
|
||||||
});
|
});
|
||||||
@ -668,27 +672,31 @@ module.exports = {
|
|||||||
now = Date.now();
|
now = Date.now();
|
||||||
|
|
||||||
let toProcess = albums.length, lastMessage = moment();
|
let toProcess = albums.length, lastMessage = moment();
|
||||||
|
return photoDB.sequelize.transaction(function(transaction) {
|
||||||
return Promise.mapSeries(albums, function(album) {
|
return Promise.mapSeries(albums, function(album) {
|
||||||
return findOrCreateDBAlbum(album).then(function() {
|
return findOrCreateDBAlbum(transaction, album).then(function() {
|
||||||
toProcess--;
|
toProcess--;
|
||||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||||
console.log("Albums to be created in DB: " + toProcess);
|
console.log("Albums to be created in DB: " + toProcess);
|
||||||
lastMessage = moment();
|
lastMessage = moment();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
});
|
||||||
}).then(function() {
|
}).then(function() {
|
||||||
console.log("Processed " + albums.length + " album DB entries in " +
|
console.log("Processed " + albums.length + " album DB entries in " +
|
||||||
((Date.now() - now) / 1000) + "s");
|
((Date.now() - now) / 1000) + "s");
|
||||||
now = Date.now();
|
now = Date.now();
|
||||||
|
|
||||||
let processed = 0, start = Date.now(), last = 0, updateScanned = [];
|
let processed = 0, start = Date.now(), last = 0, updateScanned = [];
|
||||||
|
return photoDB.sequelize.transaction(function(transaction) {
|
||||||
return Promise.map(assets, function(asset) {
|
return Promise.map(assets, function(asset) {
|
||||||
return findOrUpdateDBAsset(asset).then(function(asset) {
|
return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
|
||||||
if (asset.scanned < asset.stats.mtime) {
|
if (asset.scanned < asset.stats.mtime) {
|
||||||
needsProcessing.push(asset);
|
needsProcessing.push(asset);
|
||||||
} else {
|
} else {
|
||||||
updateScanned.push(asset.id);
|
updateScanned.push(asset.id);
|
||||||
}
|
}
|
||||||
|
});
|
||||||
}).then(function(asset) {
|
}).then(function(asset) {
|
||||||
processed++;
|
processed++;
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user