jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Scan and update looks to be working

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2018-09-29 16:40:48 -07:00
parent e0bf69faef
commit 99e16eafb6
1 changed files with 22 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
server/scanner.js
View File

@ -304,7 +304,7 @@ function processBlock(items) {
asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format();
if (asset.taken == "Invalid date" || asset.taken.replace(/T.*/, "") == "1899-11-30") {
console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
console.log("Invalid EXIF date information for " + asset.album.path + asset.filename);
asset.taken = asset.modified = moment(created).format();
}
} else {
@ -416,7 +416,7 @@ function processBlock(items) {
return;
}
return photoDB.sequelize.query("UPDATE photos SET duplicate=1,scanned=CURRENT_TIMESTAMP WHERE id IN (:dups)", {
return photoDB.sequelize.query("UPDATE photos SET duplicate=1,modified=CURRENT_TIMESTAMP,scanned=CURRENT_TIMESTAMP WHERE id IN (:dups)", {
replacements: {
dups: dups
}
@ -461,7 +461,6 @@ function processBlock(items) {
});
}
function scanDir(parent, path) {
let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
album = {
@ -584,7 +583,7 @@ function findOrCreateDBAlbum(transaction, album) {
}
function findOrUpdateDBAsset(transaction, asset) {
let query = "SELECT id FROM photos WHERE albumId=:albumId AND filename=:filename";
let query = "SELECT id,scanned,modified FROM photos WHERE albumId=:albumId AND filename=:filename";
if (!asset.album || !asset.album.id) {
let error = "Asset being processed without an album";
console.error(error);
@ -602,14 +601,14 @@ function findOrUpdateDBAsset(transaction, asset) {
replacements: asset,
transaction: transaction
}).spread(function(results, metadata) {
return [ metadata.lastID, null ];
asset.id = metadata.lastID;
});
} else {
return [ results[0].id, results[0].scanned ];
asset.id = results[0].id;
asset.scanned = results[0].scanned;
asset.modified = results[0].modified;
}
}).spread(function(id, scanned) {
asset.id = id;
asset.scanned = scanned;
}).then(function() {
return asset;
});
}
@ -682,13 +681,19 @@ module.exports = {
((Date.now() - now) / 1000) + "s");
now = Date.now();
console.log(assets.length + " assets remaining to have DB entries updated. ETA N/A");
console.log(assets.length + " assets remaining to be verified/updated. ETA N/A");
let processed = 0, start = Date.now(), last = 0, updateScanned = [];
let processed = 0, start = Date.now(), last = 0, updateScanned = [], newEntries = 0;
return photoDB.sequelize.transaction(function(transaction) {
return Promise.map(assets, function(asset) {
return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
if (asset.scanned < asset.stats.mtime || !asset.added) {
if (!asset.scanned) {
newEntries++;
if (newEntries == 5) {
console.log(asset);
}
}
if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) {
needsProcessing.push(asset);
} else {
updateScanned.push(asset.id);
@ -701,9 +706,9 @@ module.exports = {
return asset;
}
let remaining = assets.length - processed;
console.log(remaining + " assets remaining to have DB entries updated. ETA " +
Math.ceil((elapsed / 1000) * remaining / (processed - last)) + "s");
let remaining = assets.length - processed, eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
console.log(remaining + " assets remaining be verified/updated " +
"(" + newEntries + " new entries, " + (remaining - newEntries) + " up-to-date so far). ETA " + eta + "s");
last = processed;
start = Date.now();
});
@ -722,7 +727,8 @@ module.exports = {
});
}
}).then(function() {
console.log(needsProcessing.length + " assets need HASH computed");
console.log(newEntries + " assets are new. " + (needsProcessing.length - newEntries) + " assets have been modified.");
console.log(needsProcessing.length + " assets need HASH computed. " + (assets.length - needsProcessing.length) + " need no update.");;
processBlock(needsProcessing);
needsProcessing = [];
}).then(function() {