scanner::scan can now be called dynamically to re-scan
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
parent
b7459e3157
commit
085cc5a772
@ -11,7 +11,7 @@ let photoDB = null;
|
||||
|
||||
const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
|
||||
|
||||
let processQueue = [], triedClean = [], lastScan;
|
||||
let processQueue = [], triedClean = [], lastScan = new Date("1900-01-01");
|
||||
|
||||
//const rawExtension = /\.(nef|orf)$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef", "orf" ];
|
||||
|
||||
@ -220,7 +220,7 @@ function processBlock(items) {
|
||||
if (results.length == 0) {
|
||||
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
|
||||
} else if (results[0].hash != asset.hash) {
|
||||
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
|
||||
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
|
||||
} else if (results[0].photoId != asset.id) {
|
||||
console.log("Duplicate asset: " +
|
||||
"'" + asset.album.path + asset.filename + "' is a copy of " +
|
||||
@ -517,7 +517,8 @@ function scanDir(parent, path) {
|
||||
filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
|
||||
name: file.replace(/.[^.]*$/, ""),
|
||||
stats: {
|
||||
mtime: stats.mtime
|
||||
mtime: stats.mtime,
|
||||
ctime: stats.ctime
|
||||
},
|
||||
album: album
|
||||
});
|
||||
@ -573,7 +574,7 @@ function findOrCreateDBAlbum(transaction, album) {
|
||||
}
|
||||
|
||||
function findOrUpdateDBAsset(transaction, asset) {
|
||||
let query = "SELECT id,scanned,modified FROM photos WHERE albumId=:albumId AND filename=:filename";
|
||||
let query = "SELECT id,DATETIME(scanned) AS scanned,DATETIME(modified) AS modified FROM photos WHERE albumId=:albumId AND filename=:filename";
|
||||
if (!asset.album || !asset.album.id) {
|
||||
let error = "Asset being processed without an album";
|
||||
console.error(error);
|
||||
@ -595,8 +596,8 @@ function findOrUpdateDBAsset(transaction, asset) {
|
||||
});
|
||||
} else {
|
||||
asset.id = results[0].id;
|
||||
asset.scanned = results[0].scanned;
|
||||
asset.modified = results[0].modified;
|
||||
asset.scanned = new Date(results[0].scanned);
|
||||
asset.modified = new Date(results[0].modified);
|
||||
}
|
||||
}).then(function() {
|
||||
return asset;
|
||||
@ -625,34 +626,41 @@ function computeHash(filepath) {
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
scan: function (db) {
|
||||
photoDB = db;
|
||||
/* 1. Scan for all assets which will be managed by the system. readdir
|
||||
* 2. Check if entry in DB. Check mod-time in DB vs. stats from #1
|
||||
* - For albums
|
||||
* - For assets
|
||||
* 3. If not in DB, or mod-time changed, queue for HASH CHECK
|
||||
*
|
||||
* HASH CHECK
|
||||
* 1. Compute HASH
|
||||
* 2. Check for HASH in photohash -- skip?
|
||||
* 3. Check for and create thumbs/FILE thumbs/scaled/FILE
|
||||
* 4. If necessary, create JPG from RAW
|
||||
* 5. Update last-scanned date in DB for entry
|
||||
* 6. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were
|
||||
* removed on disk)? Also purge from the HASH table.
|
||||
*/
|
||||
let initialized = Date.now();
|
||||
let now = Date.now();
|
||||
let needsProcessing = [];
|
||||
function doScan() {
|
||||
/* 1. Scan for all assets which will be managed by the system. readdir
|
||||
* 2. Check if entry in DB. Check mod-time in DB vs. stats from #1
|
||||
* - For albums
|
||||
* - For assets
|
||||
* 3. If not in DB, or mod-time changed, queue for HASH CHECK
|
||||
*
|
||||
* HASH CHECK
|
||||
* 1. Compute HASH
|
||||
* 2. Check for HASH in photohash -- skip?
|
||||
* 3. Check for and create thumbs/FILE thumbs/scaled/FILE
|
||||
* 4. If necessary, create JPG from RAW
|
||||
* 5. Update last-scanned date in DB for entry
|
||||
* 6. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were
|
||||
* removed on disk)? Also purge from the HASH table.
|
||||
*/
|
||||
let initialized = Date.now();
|
||||
let now = Date.now();
|
||||
let needsProcessing = [];
|
||||
|
||||
lastScan = new Date();
|
||||
return photoDB.sequelize.query("SELECT max(scanned) AS scanned FROM photos", {
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}).then(function(results) {
|
||||
if (results[0].scanned == null) {
|
||||
lastScan = new Date("1800-01-01");
|
||||
} else {
|
||||
lastScan = new Date(results[0].scanned);
|
||||
}
|
||||
console.log("Updating any asset newer than " + moment(lastScan).format());
|
||||
}).then(function() {
|
||||
return scanDir(null, picturesPath).spread(function(albums, assets) {
|
||||
console.log("Found " + assets.length + " assets in " + albums.length + " albums after " +
|
||||
((Date.now() - now) / 1000) + "s");
|
||||
/* One at a time, in series, as the album[] array has parents first, then descendants.
|
||||
* Operating in parallel could result in a child being searched for prior to the parent */
|
||||
* Operating in parallel could result in a child being searched for prior to the parent */
|
||||
now = Date.now();
|
||||
|
||||
let toProcess = albums.length, lastMessage = moment();
|
||||
@ -676,23 +684,40 @@ module.exports = {
|
||||
let processed = 0, start = Date.now(), last = 0, updateScanned = [], newEntries = 0;
|
||||
return photoDB.sequelize.transaction(function(transaction) {
|
||||
return Promise.map(assets, function(asset) {
|
||||
return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
|
||||
if (!asset.scanned) {
|
||||
newEntries++;
|
||||
}
|
||||
if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) {
|
||||
needsProcessing.push(asset);
|
||||
} else {
|
||||
return new Promise(function(resolve, reject) {
|
||||
/* If both mtime and ctime of the asset are older than the lastScan,
|
||||
* skip this asset. */
|
||||
if (lastScan != null && asset.stats.mtime < lastScan && asset.stats.ctime < lastScan) {
|
||||
updateScanned.push(asset.id);
|
||||
return resolve(asset);
|
||||
}
|
||||
|
||||
return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
|
||||
if (!asset.scanned) {
|
||||
newEntries++;
|
||||
}
|
||||
console.log(typeof asset.scanned);
|
||||
if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) {
|
||||
console.log("Process asset: " + asset.scanned);
|
||||
needsProcessing.push(asset);
|
||||
} else {
|
||||
console.log("Update scanned tag: " + asset.scanned);
|
||||
updateScanned.push(asset.id);
|
||||
}
|
||||
return asset;
|
||||
}).then(function(asset) {
|
||||
return resolve(asset);
|
||||
}).catch(function(error) {
|
||||
return reject(error);
|
||||
});
|
||||
}).then(function(asset) {
|
||||
processed++;
|
||||
|
||||
|
||||
let elapsed = Date.now() - start;
|
||||
if (elapsed < 5000) {
|
||||
return asset;
|
||||
}
|
||||
|
||||
|
||||
let remaining = assets.length - processed, eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
|
||||
console.log(remaining + " assets remaining be verified/updated " +
|
||||
"(" + newEntries + " new entries, " + (processed - newEntries) + " up-to-date so far). ETA " + eta + "s");
|
||||
@ -727,5 +752,13 @@ module.exports = {
|
||||
}).then(function() {
|
||||
console.log("Total time to initialize DB and all scans: " + ((Date.now() - initialized) / 1000) + "s");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
scan: function (db) {
|
||||
photoDB = db;
|
||||
return doScan();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user