jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Scan and update looks to be working

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2018-09-29 16:40:48 -07:00
parent e0bf69faef
commit 99e16eafb6
1 changed files with 22 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
server/scanner.js
View File

@ -304,7 +304,7 @@ function processBlock(items) {
asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format(); asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format();
if (asset.taken == "Invalid date" || asset.taken.replace(/T.*/, "") == "1899-11-30") { if (asset.taken == "Invalid date" || asset.taken.replace(/T.*/, "") == "1899-11-30") {
console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif)); console.log("Invalid EXIF date information for " + asset.album.path + asset.filename);
asset.taken = asset.modified = moment(created).format(); asset.taken = asset.modified = moment(created).format();
} }
} else { } else {
@ -416,7 +416,7 @@ function processBlock(items) {
return; return;
} }
return photoDB.sequelize.query("UPDATE photos SET duplicate=1,scanned=CURRENT_TIMESTAMP WHERE id IN (:dups)", { return photoDB.sequelize.query("UPDATE photos SET duplicate=1,modified=CURRENT_TIMESTAMP,scanned=CURRENT_TIMESTAMP WHERE id IN (:dups)", {
replacements: { replacements: {
dups: dups dups: dups
} }
@ -461,7 +461,6 @@ function processBlock(items) {
}); });
} }
function scanDir(parent, path) { function scanDir(parent, path) {
let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"), let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
album = { album = {
@ -584,7 +583,7 @@ function findOrCreateDBAlbum(transaction, album) {
} }
function findOrUpdateDBAsset(transaction, asset) { function findOrUpdateDBAsset(transaction, asset) {
let query = "SELECT id FROM photos WHERE albumId=:albumId AND filename=:filename"; let query = "SELECT id,scanned,modified FROM photos WHERE albumId=:albumId AND filename=:filename";
if (!asset.album || !asset.album.id) { if (!asset.album || !asset.album.id) {
let error = "Asset being processed without an album"; let error = "Asset being processed without an album";
console.error(error); console.error(error);
@ -602,14 +601,14 @@ function findOrUpdateDBAsset(transaction, asset) {
replacements: asset, replacements: asset,
transaction: transaction transaction: transaction
}).spread(function(results, metadata) { }).spread(function(results, metadata) {
return [ metadata.lastID, null ]; asset.id = metadata.lastID;
}); });
} else { } else {
return [ results[0].id, results[0].scanned ]; asset.id = results[0].id;
asset.scanned = results[0].scanned;
asset.modified = results[0].modified;
} }
}).spread(function(id, scanned) { }).then(function() {
asset.id = id;
asset.scanned = scanned;
return asset; return asset;
}); });
} }
@ -682,13 +681,19 @@ module.exports = {
((Date.now() - now) / 1000) + "s"); ((Date.now() - now) / 1000) + "s");
now = Date.now(); now = Date.now();
console.log(assets.length + " assets remaining to have DB entries updated. ETA N/A"); console.log(assets.length + " assets remaining to be verified/updated. ETA N/A");
let processed = 0, start = Date.now(), last = 0, updateScanned = []; let processed = 0, start = Date.now(), last = 0, updateScanned = [], newEntries = 0;
return photoDB.sequelize.transaction(function(transaction) { return photoDB.sequelize.transaction(function(transaction) {
return Promise.map(assets, function(asset) { return Promise.map(assets, function(asset) {
return findOrUpdateDBAsset(transaction, asset).then(function(asset) { return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
if (asset.scanned < asset.stats.mtime || !asset.added) { if (!asset.scanned) {
newEntries++;
if (newEntries == 5) {
console.log(asset);
}
}
if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) {
needsProcessing.push(asset); needsProcessing.push(asset);
} else { } else {
updateScanned.push(asset.id); updateScanned.push(asset.id);
@ -701,9 +706,9 @@ module.exports = {
return asset; return asset;
} }
let remaining = assets.length - processed; let remaining = assets.length - processed, eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
console.log(remaining + " assets remaining to have DB entries updated. ETA " + console.log(remaining + " assets remaining be verified/updated " +
Math.ceil((elapsed / 1000) * remaining / (processed - last)) + "s"); "(" + newEntries + " new entries, " + (remaining - newEntries) + " up-to-date so far). ETA " + eta + "s");
last = processed; last = processed;
start = Date.now(); start = Date.now();
}); });
@ -722,7 +727,8 @@ module.exports = {
}); });
} }
}).then(function() { }).then(function() {
console.log(needsProcessing.length + " assets need HASH computed"); console.log(newEntries + " assets are new. " + (needsProcessing.length - newEntries) + " assets have been modified.");
console.log(needsProcessing.length + " assets need HASH computed. " + (assets.length - needsProcessing.length) + " need no update.");;
processBlock(needsProcessing); processBlock(needsProcessing);
needsProcessing = []; needsProcessing = [];
}).then(function() { }).then(function() {