jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Working on path structure re-creation

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2018-09-26 15:53:47 -07:00
parent 531c6db4fd
commit ae839e440b
1 changed files with 65 additions and 355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

420
server/scanner.js
View File

@ -20,11 +20,11 @@ let processQueue = [], triedClean = [];
const rawExtension = /\.nef$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef" ]; const rawExtension = /\.nef$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef" ];
function removeNewerFile(path, fileA, fileB) { function removeNewerFile(path, fileA, fileB) {
fs.stat(path + "/" + fileA, function(err, statsA) { fs.stat(path + fileA, function(err, statsA) {
if (err) { if (err) {
return; return;
} }
fs.stat(path + "/" + fileB, function(err, statsB) { fs.stat(path + fileB, function(err, statsB) {
if (err) { if (err) {
return; return;
} }
@ -39,122 +39,6 @@ function removeNewerFile(path, fileA, fileB) {
}); });
} }
function _scanDir(parent, path) {
let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
replacements = {
path: path.slice(picturesPath.length),
name: path.replace(/.*\//, "").replace(/_/g, " "),
parent: parent || null
};
/* Ensure that top level images are placed into an album with a root path */
replacements.path = replacements.path || "/";
let query = "SELECT id FROM albums WHERE path=:path AND ";
if (!parent) {
query += "parentId IS NULL";
} else {
query += "parentId=:parent";
}
return photoDB.sequelize.query(query, {
replacements: replacements,
type: photoDB.sequelize.QueryTypes.SELECT
}).then(function(results) {
if (results.length == 0) {
// console.log("Adding " + path + " under " + parent, replacements);
return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parent,:name)", {
replacements: replacements
}).spread(function(results, metadata) {
return metadata.lastID;
});
} else {
return results[0].id;
}
}).then(function(parent) {
return new Promise(function(resolve, reject) {
console.log("Scanning ..." + replacements.path);
fs.readdir(path, function(err, files) {
if (err) {
console.warn("Could not readdir " + path);
return resolve([]);
}
return resolve(files);
});
}).then(function(files) {
scanning++;
/* Remove 'thumbs' and 'raw' directories from being processed */
files = files.filter(function(file) {
for (var i = 0; i < files.length; i++) {
/* If this file has an original NEF/ORF on the system, don't add the JPG to the DB */
if (rawExtension.exec(files[i]) && file == files[i].replace(rawExtension, ".jpg")) {
return false;
}
/* If there is a different CASE (eg. JPG vs jpg) don't add it, and remove the 'lower case'
* version from disk. */
if (file != files[i] && file.toUpperCase() == files[i]) {
removeNewerFile(path, file, files[i]);
console.log("Duplicate file in " + path + ": ", file, files[i]);
return false;
}
}
return file != "raw" && file != "thumbs" && file != ".git" && file != "corrupt";
});
return mkdir(path + "/thumbs/scaled").then(function() {
return Promise.map(files, function(file) {
let filepath = path + "/" + file;
return stat(filepath).then(function(stats) {
if (stats.isDirectory()) {
return scanDir(parent, filepath).catch(function(error) {
console.warn("Could not scanDir " + filepath + ": " + error);
});
}
/* Check file extensions */
if (!re.exec(file)) {
return;
}
const replacements = {
path: path.slice(picturesPath.length),
filename: file.replace(rawExtension, ".jpg") /* We will be converting from NEF/ORF => JPG */
};
replacements.path = replacements.path || "/";
return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", {
replacements: replacements,
type: photoDB.sequelize.QueryTypes.SELECT
}).then(function(photo) {
if (photo.length == 0) {
processQueue.push([ replacements.path, file, stats.mtime, parent ]);
}
}).catch(function(error) {
console.error("Sequelize.query failed: ", error);
});
}).catch(function(error) {
console.warn("Could not stat " + filepath + ": " + error);
});
}, {
concurrency: 1
}).then(function() {
scanning--;
if (scanning == 0) {
const endStamp = Date.now();
console.log("Scanning completed in " + Math.round(((endStamp - startStamp))) + "ms. " + processQueue.length + " items to process.");
}
});
});
});
});
}
const startStamp = Date.now(); const startStamp = Date.now();
@ -179,14 +63,17 @@ const stat = function (path) {
const mkdir = function (_path) { const mkdir = function (_path) {
if (_path.indexOf(picturesPath) == 0) { if (_path.indexOf(picturesPath) == 0) {
_path = _path.substring(picturesPath.length + 1); _path = _path.substring(picturesPath.length);
} }
let parts = _path.split("/"), path; let parts = _path.split("/"), path;
console.log(parts, _path);
parts.unshift(picturesPath); parts.unshift(picturesPath);
return Promise.mapSeries(parts, function (part) { return Promise.mapSeries(parts, function (part) {
if (!path) { if (!path) {
path = picturesPath; path = picturesPath.replace(/\/$/, "");
} else { } else {
path += "/" + part; path += "/" + part;
} }
@ -217,12 +104,12 @@ const exists = function(path) {
} }
function convertRawToJpg(path, file) { function convertRawToJpg(path, file) {
console.log("Converting " + path + "/" + file); console.log("Converting " + path + file);
path = picturesPath + path; path = picturesPath + path;
return new Promise(function(resolve, reject) { return new Promise(function(resolve, reject) {
return exists(path + "/" + file.replace(rawExtension, ".jpg")).then(function(exist) { return exists(path + file.replace(rawExtension, ".jpg")).then(function(exist) {
if (exist) { if (exist) {
console.log("Skipping already converted file: " + file); console.log("Skipping already converted file: " + file);
return; return;
@ -236,8 +123,8 @@ function convertRawToJpg(path, file) {
"--compression=90", "--compression=90",
"--exif", "--exif",
"--overwrite", "--overwrite",
"--output", path + "/" + file.replace(rawExtension, ".jpg"), "--output", path + file.replace(rawExtension, ".jpg"),
path + "/" + file path + file
]); ]);
const stderr = []; const stderr = [];
@ -248,7 +135,7 @@ function convertRawToJpg(path, file) {
return new Promise(function(ufraw, resolve, reject) { return new Promise(function(ufraw, resolve, reject) {
ufraw.on('exit', function(stderr, code, signal) { ufraw.on('exit', function(stderr, code, signal) {
if (signal || code != 0) { if (signal || code != 0) {
let error = "UFRAW for " + path + "/" + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n"; let error = "UFRAW for " + path + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n";
console.error(error); console.error(error);
return moveCorrupt(path, file).then(function() { return moveCorrupt(path, file).then(function() {
console.warn("ufraw failed"); console.warn("ufraw failed");
@ -258,10 +145,10 @@ function convertRawToJpg(path, file) {
return reject(error); return reject(error);
}); });
} }
return mkdir(path + "/raw").then(function() { return mkdir(path + "raw").then(function() {
fs.rename(path + "/" + file, path + "/raw/" + file, function(err) { fs.rename(path + file, path + "raw/" + file, function(err) {
if (err) { if (err) {
console.error("Unable to move RAW file: " + path + "/" + file); console.error("Unable to move RAW file: " + path + file);
return reject(err); return reject(err);
} }
return resolve(); return resolve();
@ -281,13 +168,13 @@ function moveCorrupt(path, file) {
path = picturesPath + path; path = picturesPath + path;
} }
console.warn("Moving corrupt file '" + file + "' to " + path + "/corrupt"); console.warn("Moving corrupt file '" + file + "' to " + path + "corrupt");
return mkdir(path + "/corrupt").then(function() { return mkdir(path + "corrupt").then(function() {
return new Promise(function(resolve, reject) { return new Promise(function(resolve, reject) {
fs.rename(path + "/" + file, path + "/corrupt/" + file, function(err) { fs.rename(path + file, path + "corrupt/" + file, function(err) {
if (err) { if (err) {
console.error("Unable to move corrupt file: " + path + "/" + file); console.error("Unable to move corrupt file: " + path + file);
return reject(err); return reject(err);
} }
return resolve(); return resolve();
@ -296,216 +183,40 @@ function moveCorrupt(path, file) {
}); });
} }
function triggerWatcher() {
setTimeout(triggerWatcher, 1000);
if (!processRunning && processQueue.length) {
let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0);
processRunning = true;
/* Sort to newest files to be processed first */
processing.sort(function(a, b) {
return b[2] - a[2];
});
return Promise.map(processing, function(entry) {
var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3];
let tmp = Promise.resolve(file);
/* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */
if (rawExtension.exec(file)) {
tmp = exists(picturesPath + path + "/" + file.replace(rawExtension, ".jpg")).then(function(exist) {
if (exist) {
return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
}
return mkdir(picturesPath + path + "/raw").then(function() {
return convertRawToJpg(path, file);
}).then(function() {
return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
});
});
}
return tmp.then(function(file) {
var src = picturesPath + path + "/" + file,
image = sharp(src);
return image.limitInputPixels(1073741824).metadata().then(function(metadata) {
if (metadata.exif) {
metadata.exif = exif(metadata.exif);
delete metadata.exif.thumbnail;
delete metadata.exif.image;
for (var key in metadata.exif.exif) {
if (Buffer.isBuffer(metadata.exif.exif[key])) {
metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
}
}
}
let replacements = {
albumId: albumId,
name: file.replace(/.[^.]*$/, ""),
path: path,
filename: file,
width: metadata.width,
height: metadata.height,
added: moment().format()
};
/* Ensure that top level images are placed into an album with a root path */
replacements.path = replacements.path || "/";
if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
replacements.taken = moment(metadata.exif.exif.DateTimeOriginal).format();
replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format();
if (replacements.taken == "Invalid date" || replacements.taken.replace(/T.*/, "") == "1899-11-30") {
console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
replacements.taken = replacements.modified = moment(created).format();
}
} else {
/* Attempt to infer the datestamp from the filename */
let date = moment(created).format();
let match = file.match(/WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/);
if (match) {
date = moment((match[1]+" "+match[2]), "YYYY-MM-DD h.mm.ss a").format();
if (date == "Invalid date") {
date = moment(created).format();
}
} else {
match = file.match(/(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/);
if (match) {
if (match[2]) { /* Stamp had time in it */
date = moment((match[1]+""+match[2]).replace(/-/g, ""), "YYYYMMDDHHmmss").format();
} else {
date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
}
if (date == "Invalid date") {
date = moment(created).format();
}
} else {
date = moment(created).format();
}
}
replacements.taken = replacements.modified = date;
}
let dst = picturesPath + path + "/thumbs/" + file;
return exists(dst).then(function(exist) {
if (exist) {
return;
}
return image.resize(256, 256).toFile(dst).catch(function(error) {
console.error("Error resizing image: " + dst, error);
throw error;
});
}).then(function() {
let dst = picturesPath + path + "/thumbs/scaled/" + file;
return exists(dst).then(function(exist) {
if (exist) {
return;
}
return image.resize(Math.min(1024, metadata.width)).toFile(dst).catch(function(error) {
console.error("Error resizing image: " + dst, error);
throw error;
});
});
}).then(function() {
return photoDB.sequelize.query("INSERT INTO photos " +
"(albumId,path,filename,added,modified,taken,width,height,name)" +
"VALUES(:albumId,:path,:filename,DATETIME(:added),DATETIME(:modified),DATETIME(:taken),:width,:height,:name)", {
replacements: replacements
});
}).then(function() {
toProcess--;
if (moment().add(-5, 'seconds') > lastMessage) {
console.log("Items to be processed: " + toProcess);
lastMessage = moment();
}
});
}).catch(function(error) {
console.error("Error reading image " + src + ": ", error);
return moveCorrupt(path, file).then(function() {
/* If the original file was not a NEF/ORF, we are done... */
if (!rawExtension.exec(entry[1])) {
return;
}
/* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */
for (var i = 0; i < triedClean.length; i++) {
if (triedClean[i] == path + "/" + file) {
/* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */
// return moveCorrupt(path, entry[1]);
console.error("Already attempted to convert NEF/ORF to JPG: " + path + "/" + file);
return;
}
}
console.warn("Adding " + path + "/" + file + " back onto processing queue.");
triedClean.push(path + "/" + file);
processQueue.push([ path, file, created, albumId ]);
});
});
}).catch(function() {
console.warn("Continuing file processing.");
});
}, {
concurrency: 1
}).then(function() {
console.log("Completed processing queue.");
});
}
}
/*******************************************************************************************************/ /*******************************************************************************************************/
let processTimeout = null;
function processBlock(items) { function processBlock(items) {
if (!processTimeout) { /* Invoke once per second to check if there are items to process */
processTimeout = setTimeout(processBlock, 1000); setTimeout(processBlock, 1000);
}
processTimeout = null;
if (items) { if (items) {
processQueue = processQueue.concat(items); processQueue = processQueue.concat(items);
} }
return;
/* 'needsProcessing' should only have their scanned stamp updated once thumbnails
* have been created. */
console.log(needsProcessing.length + " had their HASH updated. Updating scanned stamp.");
return db.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:scanned)", {
replacements: {
scanned: needsProcessing
}
});
if (!processRunning && processQueue.length) { if (!processRunning && processQueue.length) {
let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0); let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0);
processRunning = true; processRunning = true;
console.log(processQueue.length);
/* Sort to newest files to be processed first */ /* Sort to newest files to be processed first */
processing.sort(function(a, b) { processing.sort(function(a, b) {
return b[2] - a[2]; return a.stats.mtime - b.stats.mtime;
}); });
return Promise.map(processing, function(entry) { return Promise.map(processing, function(asset) {
var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3]; var path = asset.album.path,
file = asset.filename,
created = asset.stats.ctime,
albumId = asset.album.id;
console.log(picturesPath, path, file);
let tmp = Promise.resolve(file); let tmp = Promise.resolve(file);
/* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */ /* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */
if (rawExtension.exec(file)) { if (rawExtension.exec(file)) {
tmp = exists(picturesPath + path + "/" + file.replace(rawExtension, ".jpg")).then(function(exist) { tmp = exists(picturesPath + path + file.replace(rawExtension, ".jpg")).then(function(exist) {
if (exist) { if (exist) {
return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */ return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
} }
return mkdir(picturesPath + path + "/raw").then(function() { return mkdir(picturesPath + path + "raw").then(function() {
return convertRawToJpg(path, file); return convertRawToJpg(path, file);
}).then(function() { }).then(function() {
return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */ return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
@ -514,7 +225,7 @@ console.log(processQueue.length);
} }
return tmp.then(function(file) { return tmp.then(function(file) {
var src = picturesPath + path + "/" + file, var src = picturesPath + path + file,
image = sharp(src); image = sharp(src);
return image.limitInputPixels(1073741824).metadata().then(function(metadata) { return image.limitInputPixels(1073741824).metadata().then(function(metadata) {
@ -529,26 +240,17 @@ console.log(processQueue.length);
} }
} }
let replacements = { asset.width = metadata.width;
albumId: albumId, asset.height = metadata.height;
name: file.replace(/.[^.]*$/, ""), asset.added = moment().format();
path: path,
filename: file,
width: metadata.width,
height: metadata.height,
added: moment().format()
};
/* Ensure that top level images are placed into an album with a root path */
replacements.path = replacements.path || "/";
if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) { if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
replacements.taken = moment(metadata.exif.exif.DateTimeOriginal).format(); asset.taken = moment(metadata.exif.exif.DateTimeOriginal).format();
replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format(); asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format();
if (replacements.taken == "Invalid date" || replacements.taken.replace(/T.*/, "") == "1899-11-30") { if (asset.taken == "Invalid date" || asset.taken.replace(/T.*/, "") == "1899-11-30") {
console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif)); console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
replacements.taken = replacements.modified = moment(created).format(); asset.taken = asset.modified = moment(created).format();
} }
} else { } else {
/* Attempt to infer the datestamp from the filename */ /* Attempt to infer the datestamp from the filename */
@ -575,10 +277,10 @@ console.log(processQueue.length);
date = moment(created).format(); date = moment(created).format();
} }
} }
replacements.taken = replacements.modified = date; asset.taken = asset.modified = date;
} }
let dst = picturesPath + path + "/thumbs/" + file; let dst = picturesPath + path + "thumbs/" + file;
return exists(dst).then(function(exist) { return exists(dst).then(function(exist) {
if (exist) { if (exist) {
@ -590,7 +292,7 @@ console.log(processQueue.length);
throw error; throw error;
}); });
}).then(function() { }).then(function() {
let dst = picturesPath + path + "/thumbs/scaled/" + file; let dst = picturesPath + path + "thumbs/scaled/" + file;
return exists(dst).then(function(exist) { return exists(dst).then(function(exist) {
if (exist) { if (exist) {
return; return;
@ -602,10 +304,10 @@ console.log(processQueue.length);
}); });
}); });
}).then(function() { }).then(function() {
return photoDB.sequelize.query("INSERT INTO photos " + return photoDB.sequelize.query("UPDATE photos SET " +
"(albumId,path,filename,added,modified,taken,width,height,name)" + "(added,modified,taken,width,height,scanned)" +
"VALUES(:albumId,:path,:filename,DATETIME(:added),DATETIME(:modified),DATETIME(:taken),:width,:height,:name)", { "VALUES(:added,:modified,:taken,:width,:height,CURRENT_TIMESTAMP)", {
replacements: replacements replacements: asset
}); });
}).then(function() { }).then(function() {
toProcess--; toProcess--;
@ -619,23 +321,23 @@ console.log(processQueue.length);
return moveCorrupt(path, file).then(function() { return moveCorrupt(path, file).then(function() {
/* If the original file was not a NEF/ORF, we are done... */ /* If the original file was not a NEF/ORF, we are done... */
if (!rawExtension.exec(entry[1])) { if (!rawExtension.exec(asset.filename)) {
return; return;
} }
/* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */ /* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */
for (var i = 0; i < triedClean.length; i++) { for (var i = 0; i < triedClean.length; i++) {
if (triedClean[i] == path + "/" + file) { if (triedClean[i] == path + file) {
/* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */ /* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */
// return moveCorrupt(path, entry[1]); // return moveCorrupt(path, asset.filename);
console.error("Already attempted to convert NEF/ORF to JPG: " + path + "/" + file); console.error("Already attempted to convert NEF/ORF to JPG: " + path + file);
return; return;
} }
} }
console.warn("Adding " + path + "/" + file + " back onto processing queue."); console.warn("Adding " + path + file + " back onto processing queue.");
triedClean.push(path + "/" + file); triedClean.push(path + file);
processQueue.push([ path, file, created, albumId ]); processQueue.push([ path, file, created, albumId ]);
}); });
}); });
@ -654,7 +356,7 @@ console.log(processQueue.length);
function scanDir(parent, path) { function scanDir(parent, path) {
let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"), let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
album = { album = {
path: path.slice(picturesPath.length) + "/", path: path.slice(picturesPath.length), /* path already ends in '/' */
name: path.replace(/.*\//, "").replace(/_/g, " "), name: path.replace(/.*\//, "").replace(/_/g, " "),
parent: parent, parent: parent,
allAssetCount: 0, allAssetCount: 0,
@ -692,10 +394,10 @@ function scanDir(parent, path) {
}); });
}).then(function(files) { }).then(function(files) {
return Promise.map(files, function(file) { return Promise.map(files, function(file) {
let filepath = path + "/" + file; let filepath = path + file;
return stat(filepath).then(function(stats) { return stat(filepath).then(function(stats) {
if (stats.isDirectory()) { if (stats.isDirectory()) {
filepath += "/";
return scanDir(album, filepath).spread(function(_albums, _assets) { return scanDir(album, filepath).spread(function(_albums, _assets) {
album.allAssetCount += _assets.length; album.allAssetCount += _assets.length;
album.allAlbumCount += _albums.length + 1; album.allAlbumCount += _albums.length + 1;
@ -710,7 +412,9 @@ function scanDir(parent, path) {
if (!re.exec(file)) { if (!re.exec(file)) {
return; return;
} }
album.hasAssets = true;
assets.push({ assets.push({
filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */ filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
name: file.replace(/.[^.]*$/, ""), name: file.replace(/.[^.]*$/, ""),
@ -719,6 +423,12 @@ function scanDir(parent, path) {
}); });
}); });
}); });
}).then(function() {
return Promise.map(albums, function(album) {
if (album.hasAssets) {
return mkdir(album.path + "thumbs");
}
});
}).then(function() { }).then(function() {
return [ albums, assets ]; return [ albums, assets ];
}); });
@ -904,7 +614,7 @@ module.exports = {
replacements: asset replacements: asset
}).then(function() { }).then(function() {
/* HASH has been updated; add to the needsProcessing array */ /* HASH has been updated; add to the needsProcessing array */
needsProcessing.push(asset.id); needsProcessing.push(asset);
}); });
}); });
}).then(function() { }).then(function() {