jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Re-load continues processing correctly

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2018-09-29 17:27:14 -07:00
parent 7fb65d04d5
commit 88e3a284b7
1 changed files with 17 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
server/scanner.js
View File

@ -202,8 +202,8 @@ function processBlock(items) {
}); });
let toProcess = processing.length, lastMessage = moment(); let toProcess = processing.length, lastMessage = moment();
console.log("Hash items to be processed: " + toProcess); console.log("Items to be processed: " + toProcess);
return Promise.map(processing, function(asset) { return Promise.mapSeries(processing, function(asset) {
return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) { return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
asset.hash = hash; asset.hash = hash;
return asset; return asset;
@ -226,7 +226,7 @@ function processBlock(items) {
"'" + asset.album.path + asset.filename + "' is a copy of " + "'" + asset.album.path + asset.filename + "' is a copy of " +
"'" + results[0].path + results[0].filename + "'"); "'" + results[0].path + results[0].filename + "'");
duplicates.push(asset); duplicates.push(asset);
return; return null;
} }
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */ /* Even if the hash doesn't need to be updated, the entry needs to be scanned */
@ -241,23 +241,12 @@ function processBlock(items) {
}).then(function() { }).then(function() {
return asset; return asset;
}); });
}).then(function() { });
toProcess--; }).then(function(asset) {
if (moment().add(-5, 'seconds') > lastMessage) { if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */
console.log("Hash items to be processed: " + toProcess); return;
lastMessage = moment();
} }
});
});
}, {
concurrency: 5
}).catch(function(error) {
console.log("Error commiting HASH transactions");
throw error;
}).then(function() {
let toProcess = needsProcessing.length, lastMessage = moment();
console.log(needsProcessing.length + " assets need to have metadata extracted");
return Promise.map(needsProcessing, function(asset) {
var path = asset.album.path, var path = asset.album.path,
file = asset.filename, file = asset.filename,
created = asset.stats.mtime, created = asset.stats.mtime,
@ -364,12 +353,6 @@ function processBlock(items) {
"WHERE id=:id", { "WHERE id=:id", {
replacements: asset, replacements: asset,
}); });
}).then(function() {
toProcess--;
if (moment().add(-5, 'seconds') > lastMessage) {
console.log("Items to be processed: " + toProcess);
lastMessage = moment();
}
}); });
}).catch(function(error) { }).catch(function(error) {
console.error("Error reading image " + src + ": ", error); console.error("Error reading image " + src + ": ", error);
@ -399,9 +382,16 @@ function processBlock(items) {
}).catch(function() { }).catch(function() {
console.warn("Continuing file processing."); console.warn("Continuing file processing.");
}); });
}, { }).then(function() {
concurrency: 5 toProcess--;
if (moment().add(-5, 'seconds') > lastMessage) {
console.log("Items to be processed: " + toProcess);
lastMessage = moment();
}
}); });
}).catch(function(error) {
console.log("Error processing file. Continuing.");
throw error;
}).then(function() { }).then(function() {
console.log("Completed processing queue. Marking " + duplicates.length + " duplicates."); console.log("Completed processing queue. Marking " + duplicates.length + " duplicates.");
let dups = []; let dups = [];