Ready to plumb in processBlock to do the thumbnailing
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
parent
92b0263396
commit
531c6db4fd
@ -113,7 +113,7 @@ router.get("/*", function(req, res/*, next*/) {
|
||||
}
|
||||
|
||||
let path = decodeURI(req.url).replace(/\?.*$/, ""),
|
||||
query = "SELECT * FROM photos WHERE path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
|
||||
query = "SELECT photos.* FROM photos INNER JOIN albums ON albums.id=photos.albumId AND albums.path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
|
||||
|
||||
console.log("Fetching from: " + path);
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@ let photoDB = null;
|
||||
|
||||
const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
|
||||
|
||||
const processQueue = [], triedClean = [];
|
||||
let processQueue = [], triedClean = [];
|
||||
|
||||
//const rawExtension = /\.(nef|orf)$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef", "orf" ];
|
||||
|
||||
@ -464,6 +464,193 @@ function triggerWatcher() {
|
||||
}
|
||||
}
|
||||
/*******************************************************************************************************/
|
||||
let processTimeout = null;
|
||||
function processBlock(items) {
|
||||
if (!processTimeout) {
|
||||
processTimeout = setTimeout(processBlock, 1000);
|
||||
}
|
||||
processTimeout = null;
|
||||
|
||||
if (items) {
|
||||
processQueue = processQueue.concat(items);
|
||||
}
|
||||
|
||||
return;
|
||||
/* 'needsProcessing' should only have their scanned stamp updated once thumbnails
|
||||
* have been created. */
|
||||
console.log(needsProcessing.length + " had their HASH updated. Updating scanned stamp.");
|
||||
return db.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:scanned)", {
|
||||
replacements: {
|
||||
scanned: needsProcessing
|
||||
}
|
||||
});
|
||||
|
||||
if (!processRunning && processQueue.length) {
|
||||
let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0);
|
||||
processRunning = true;
|
||||
console.log(processQueue.length);
|
||||
/* Sort to newest files to be processed first */
|
||||
processing.sort(function(a, b) {
|
||||
return b[2] - a[2];
|
||||
});
|
||||
|
||||
return Promise.map(processing, function(entry) {
|
||||
var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3];
|
||||
|
||||
let tmp = Promise.resolve(file);
|
||||
/* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */
|
||||
if (rawExtension.exec(file)) {
|
||||
tmp = exists(picturesPath + path + "/" + file.replace(rawExtension, ".jpg")).then(function(exist) {
|
||||
if (exist) {
|
||||
return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
|
||||
}
|
||||
|
||||
return mkdir(picturesPath + path + "/raw").then(function() {
|
||||
return convertRawToJpg(path, file);
|
||||
}).then(function() {
|
||||
return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return tmp.then(function(file) {
|
||||
var src = picturesPath + path + "/" + file,
|
||||
image = sharp(src);
|
||||
|
||||
return image.limitInputPixels(1073741824).metadata().then(function(metadata) {
|
||||
if (metadata.exif) {
|
||||
metadata.exif = exif(metadata.exif);
|
||||
delete metadata.exif.thumbnail;
|
||||
delete metadata.exif.image;
|
||||
for (var key in metadata.exif.exif) {
|
||||
if (Buffer.isBuffer(metadata.exif.exif[key])) {
|
||||
metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let replacements = {
|
||||
albumId: albumId,
|
||||
name: file.replace(/.[^.]*$/, ""),
|
||||
path: path,
|
||||
filename: file,
|
||||
width: metadata.width,
|
||||
height: metadata.height,
|
||||
added: moment().format()
|
||||
};
|
||||
|
||||
/* Ensure that top level images are placed into an album with a root path */
|
||||
replacements.path = replacements.path || "/";
|
||||
|
||||
if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
|
||||
replacements.taken = moment(metadata.exif.exif.DateTimeOriginal).format();
|
||||
replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format();
|
||||
|
||||
if (replacements.taken == "Invalid date" || replacements.taken.replace(/T.*/, "") == "1899-11-30") {
|
||||
console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
|
||||
replacements.taken = replacements.modified = moment(created).format();
|
||||
}
|
||||
} else {
|
||||
/* Attempt to infer the datestamp from the filename */
|
||||
let date = moment(created).format();
|
||||
|
||||
let match = file.match(/WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/);
|
||||
if (match) {
|
||||
date = moment((match[1]+" "+match[2]), "YYYY-MM-DD h.mm.ss a").format();
|
||||
if (date == "Invalid date") {
|
||||
date = moment(created).format();
|
||||
}
|
||||
} else {
|
||||
match = file.match(/(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/);
|
||||
if (match) {
|
||||
if (match[2]) { /* Stamp had time in it */
|
||||
date = moment((match[1]+""+match[2]).replace(/-/g, ""), "YYYYMMDDHHmmss").format();
|
||||
} else {
|
||||
date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
|
||||
}
|
||||
if (date == "Invalid date") {
|
||||
date = moment(created).format();
|
||||
}
|
||||
} else {
|
||||
date = moment(created).format();
|
||||
}
|
||||
}
|
||||
replacements.taken = replacements.modified = date;
|
||||
}
|
||||
|
||||
let dst = picturesPath + path + "/thumbs/" + file;
|
||||
|
||||
return exists(dst).then(function(exist) {
|
||||
if (exist) {
|
||||
return;
|
||||
}
|
||||
|
||||
return image.resize(256, 256).toFile(dst).catch(function(error) {
|
||||
console.error("Error resizing image: " + dst, error);
|
||||
throw error;
|
||||
});
|
||||
}).then(function() {
|
||||
let dst = picturesPath + path + "/thumbs/scaled/" + file;
|
||||
return exists(dst).then(function(exist) {
|
||||
if (exist) {
|
||||
return;
|
||||
}
|
||||
|
||||
return image.resize(Math.min(1024, metadata.width)).toFile(dst).catch(function(error) {
|
||||
console.error("Error resizing image: " + dst, error);
|
||||
throw error;
|
||||
});
|
||||
});
|
||||
}).then(function() {
|
||||
return photoDB.sequelize.query("INSERT INTO photos " +
|
||||
"(albumId,path,filename,added,modified,taken,width,height,name)" +
|
||||
"VALUES(:albumId,:path,:filename,DATETIME(:added),DATETIME(:modified),DATETIME(:taken),:width,:height,:name)", {
|
||||
replacements: replacements
|
||||
});
|
||||
}).then(function() {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
console.log("Items to be processed: " + toProcess);
|
||||
lastMessage = moment();
|
||||
}
|
||||
});
|
||||
}).catch(function(error) {
|
||||
console.error("Error reading image " + src + ": ", error);
|
||||
return moveCorrupt(path, file).then(function() {
|
||||
|
||||
/* If the original file was not a NEF/ORF, we are done... */
|
||||
if (!rawExtension.exec(entry[1])) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */
|
||||
for (var i = 0; i < triedClean.length; i++) {
|
||||
if (triedClean[i] == path + "/" + file) {
|
||||
/* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */
|
||||
// return moveCorrupt(path, entry[1]);
|
||||
|
||||
console.error("Already attempted to convert NEF/ORF to JPG: " + path + "/" + file);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
console.warn("Adding " + path + "/" + file + " back onto processing queue.");
|
||||
triedClean.push(path + "/" + file);
|
||||
processQueue.push([ path, file, created, albumId ]);
|
||||
});
|
||||
});
|
||||
}).catch(function() {
|
||||
console.warn("Continuing file processing.");
|
||||
});
|
||||
}, {
|
||||
concurrency: 1
|
||||
}).then(function() {
|
||||
console.log("Completed processing queue.");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function scanDir(parent, path) {
|
||||
let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
|
||||
album = {
|
||||
@ -650,6 +837,7 @@ module.exports = {
|
||||
*/
|
||||
let initialized = Date.now();
|
||||
let now = Date.now();
|
||||
const needsProcessing = [], duplicates = [];
|
||||
return scanDir(null, picturesPath).spread(function(albums, assets) {
|
||||
console.log("Found " + assets.length + " assets in " + albums.length + " albums after " +
|
||||
((Date.now() - now) / 1000) + "s");
|
||||
@ -679,7 +867,7 @@ module.exports = {
|
||||
}
|
||||
|
||||
let remaining = assets.length - processed;
|
||||
console.log(remaining + " assets remaining. ETA " +
|
||||
console.log(remaining + " assets remaining to have DB entries updated. ETA " +
|
||||
Math.ceil((elapsed / 1000) * remaining / (processed - last)) + "s");
|
||||
last = processed;
|
||||
start = Date.now();
|
||||
@ -693,9 +881,9 @@ module.exports = {
|
||||
asset.hash = hash;
|
||||
});
|
||||
}).then(function() {
|
||||
let needsProcessing = [];
|
||||
return Promise.map(hashNeeded, function(asset) {
|
||||
return db.sequelize.query("SELECT hash FROM photohashes WHERE photoId=:id", {
|
||||
/* Needs to be one at a time in case there are multiple HASH collisions */
|
||||
return Promise.mapSeries(hashNeeded, function(asset) {
|
||||
return db.sequelize.query("SELECT * FROM photohashes WHERE hash=:hash OR photoId=:id", {
|
||||
replacements: asset,
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}).then(function(results) {
|
||||
@ -704,6 +892,10 @@ module.exports = {
|
||||
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
|
||||
} else if (results[0].hash != asset.hash) {
|
||||
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
|
||||
} else if (results[0].photoId != asset.id) {
|
||||
console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename);
|
||||
duplicates.push(asset);
|
||||
return;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
@ -715,29 +907,16 @@ module.exports = {
|
||||
needsProcessing.push(asset.id);
|
||||
});
|
||||
});
|
||||
}, {
|
||||
concurrency: 5
|
||||
}).then(function() {
|
||||
if (!needsProcessing.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* 'needsProcessing' should only have their scanned stamp updated once thumbnails
|
||||
* have been created. */
|
||||
console.log(needsProcessing.length + " had their HASH updated. Updating scanned stamp.");
|
||||
return db.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:scanned)", {
|
||||
replacements: {
|
||||
scanned: needsProcessing
|
||||
}
|
||||
});
|
||||
processBlock(needsProcessing);
|
||||
});
|
||||
});
|
||||
}).then(function() {
|
||||
console.log("Processed " + assets.length + " asset DB entries in " +
|
||||
((Date.now() - now) / 1000) + "s");
|
||||
console.log(duplicates.length + " duplicates.");
|
||||
});
|
||||
});
|
||||
/*triggerWatcher();*/
|
||||
}).then(function() {
|
||||
console.log("Total time to initialize DB and all scans: " + ((Date.now() - initialized) / 1000) + "s");
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user