jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Fixed concurrency on hash creation.

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2018-09-29 16:15:14 -07:00
parent 4f1d72147a
commit e0bf69faef
1 changed files with 50 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
server/scanner.js
View File

@ -209,51 +209,51 @@ function processBlock(items) {
return asset;
}).then(function(asset) {
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
"WHERE hash=:hash OR photoId=:id", {
replacements: asset,
type: photoDB.sequelize.QueryTypes.SELECT
}).then(function(results) {
let query;
if (results.length == 0) {
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
} else if (results[0].hash != asset.hash) {
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
} else if (results[0].photoId != asset.id) {
console.log("Duplicate asset: " +
"'" + asset.album.path + asset.filename + "' is a copy of " +
"'" + results[0].path + results[0].filename + "'");
duplicates.push(asset);
return;
}
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
needsProcessing.push(asset);
if (!query) {
return asset;
}
return photoDB.sequelize.query(query, {
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
"WHERE hash=:hash OR photoId=:id", {
replacements: asset,
type: photoDB.sequelize.QueryTypes.SELECT
}).then(function(results) {
let query;
if (results.length == 0) {
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
} else if (results[0].hash != asset.hash) {
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
} else if (results[0].photoId != asset.id) {
console.log("Duplicate asset: " +
"'" + asset.album.path + asset.filename + "' is a copy of " +
"'" + results[0].path + results[0].filename + "'");
duplicates.push(asset);
return;
}
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
needsProcessing.push(asset);
if (!query) {
return asset;
}
return photoDB.sequelize.query(query, {
replacements: asset,
}).then(function() {
return asset;
});
}).then(function() {
return asset;
toProcess--;
if (moment().add(-5, 'seconds') > lastMessage) {
console.log("Hash items to be processed: " + toProcess);
lastMessage = moment();
}
});
}).then(function() {
toProcess--;
if (moment().add(-5, 'seconds') > lastMessage) {
console.log("Hash items to be processed: " + toProcess);
lastMessage = moment();
}
});
}, {
concurrency: 5
}).catch(function(error) {
console.log("Error commiting HASH transactions");
throw error;
});
}).then(function() {
let toProcess = needsProcessing.length, lastMessage = moment();
console.log(needsProcessing.length + " assets need to have metadata extracted");
@ -682,7 +682,7 @@ module.exports = {
((Date.now() - now) / 1000) + "s");
now = Date.now();
console.log(albums.length + " assets remaining to have DB entries updated. ETA N/A");
console.log(assets.length + " assets remaining to have DB entries updated. ETA N/A");
let processed = 0, start = Date.now(), last = 0, updateScanned = [];
return photoDB.sequelize.transaction(function(transaction) {
@ -693,22 +693,22 @@ module.exports = {
} else {
updateScanned.push(asset.id);
}
}).then(function(asset) {
processed++;
let elapsed = Date.now() - start;
if (elapsed < 5000) {
return asset;
}
let remaining = assets.length - processed;
console.log(remaining + " assets remaining to have DB entries updated. ETA " +
Math.ceil((elapsed / 1000) * remaining / (processed - last)) + "s");
last = processed;
start = Date.now();
});
}, {
concurrency: 10
}).then(function(asset) {
processed++;
let elapsed = Date.now() - start;
if (elapsed < 5000) {
return asset;
}
let remaining = assets.length - processed;
console.log(remaining + " assets remaining to have DB entries updated. ETA " +
Math.ceil((elapsed / 1000) * remaining / (processed - last)) + "s");
last = processed;
start = Date.now();
});
}).then(function() {
if (updateScanned.length) {