diff --git a/package.json b/package.json
index 2419cca..93316ff 100644
--- a/package.json
+++ b/package.json
@@ -29,7 +29,8 @@
"qs": "^6.5.2",
"sequelize": "^4.28.6",
"sequelize-mysql": "^1.7.0",
- "sharp": "^0.20.5"
+ "sharp": "^0.20.5",
+ "sqlite3": "^4.0.2"
},
"jshintConfig": {
"undef": true,
diff --git a/server/scanner.js b/server/scanner.js
index a8bcd6c..897daa2 100644
--- a/server/scanner.js
+++ b/server/scanner.js
@@ -11,22 +11,29 @@ let photoDB = null;
const picturesPath = config.get("picturesPath");
-const processQueue = [];
+const processQueue = [], triedClean = [];
function scanDir(parent, path) {
- let extensions = [ "jpg", "jpeg", "png", "gif" ],
- re = "\.((" + extensions.join(")|(") + "))$";
- re = new RegExp(re, "i");
-
- return photoDB.sequelize.query("SELECT id FROM albums WHERE path=:path AND parentId=:parent", {
- replacements: {
+ let extensions = [ "jpg", "jpeg", "png", "gif", "nef" ],
+ re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
+ replacements = {
path: path,
parent: parent || null
- },
+ };
+
+ let query = "SELECT id FROM albums WHERE path=:path AND ";
+ if (!parent) {
+ query += "parentId IS NULL";
+ } else {
+ query += "parentId=:parent";
+ }
+
+ return photoDB.sequelize.query(query, {
+ replacements: replacements,
type: photoDB.sequelize.QueryTypes.SELECT
}).then(function(results) {
if (results.length == 0) {
- console.log("Adding " + path + " under " + parent);
+// console.log("Adding " + path + " under " + parent, replacements);
return photoDB.sequelize.query("INSERT INTO albums SET path=:path,parentId=:parent", {
replacements: {
path: path,
@@ -40,7 +47,7 @@ function scanDir(parent, path) {
}
}).then(function(parent) {
return new Promise(function(resolve, reject) {
- // console.log("Scanning path " + path);
+ console.log("Scanning path " + path + " under parent " + parent);
fs.readdir(path, function(err, files) {
if (err) {
@@ -57,26 +64,31 @@ function scanDir(parent, path) {
break;
}
}
-
- let tmp = Promise.resolve();
-
+
+ let tmp;
if (!hasThumbs) {
- tmp = new Promise(function(resolve, reject) {
- fs.mkdir(path + "/thumbs", function(err) {
- if (err) {
- return reject("Unable to create " + paths + "/thumbs");
- }
- return resolve();
- });
- });
+ tmp = mkdirPromise(path + "/thumbs");
+ } else {
+ tmp = Promise.resolve();
}
+ /* Remove 'thumbs' and 'raw' directories from being processed */
+ files = files.filter(function(file) {
+ for (var i = 0; i < files.length; i++) {
+ /* If this file has an original NEF on the system, don't add the JPG to the
+ * DB */
+ if (/\.nef$/i.exec(files[i]) && file == files[i].replace(/\.nef$/i, ".jpg")) {
+ console.log("Skipping DB duplicate for " + files[i]);
+ return false;
+ }
+ }
+
+ return file != "raw" && file != "thumbs" && file != ".git" && file != "corrupt";
+ });
+
return tmp.then(function() {
return Promise.map(files, function(file) {
let filepath = path + "/" + file;
- if (file == "thumbs") {
- return resolve(true);
- }
return new Promise(function(resolve, reject) {
fs.stat(filepath, function(err, stats) {
@@ -91,14 +103,15 @@ function scanDir(parent, path) {
});
}
- /* stats.isFile() */
+ /* Check file extensions */
if (!re.exec(file)) {
return resolve(true);
}
+
const replacements = {
path: path.slice(picturesPath.length),
- filename: file
+ filename: file.replace(/\.nef$/i, ".jpg") /* We will be converting from NEF => JPG */
};
return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", {
@@ -137,81 +150,204 @@ const { spawn } = require('child_process');
const sharp = require("sharp"), exif = require("exif-reader");
+function mkdirPromise(path) {
+ return new Promise(function(resolve, reject) {
+ fs.stat(path, function(err, stats) {
+ if (err && err.code != 'ENOENT') {
+ console.warn("Could not stat " + path + "/raw");
+ return reject();
+ }
+
+ if (!err) {
+ return resolve();
+ }
+
+ fs.mkdir(path, function(err) {
+ if (err) {
+ return reject("Unable to create " + path, err);
+ }
+ return resolve();
+ });
+ });
+ });
+}
+
+function convertNefToJpg(path, file) {
+ console.log("Converting " + path + "/" + file);
+
+ path = picturesPath + path;
+ return new Promise(function(resolve, reject) {
+ fs.stat(path + "/" + file.replace(/\.nef$/i, ".jpg"), function(err, stats) {
+ if (!err) {
+ console.log("Skipping already converted file: " + file);
+ return resolve();
+ }
+
+ const ufraw = spawn("ufraw-batch", [
+ "--silent",
+ "--wb=camera",
+ "--rotate=camera",
+ "--out-type=jpg",
+ "--compression=90",
+ "--exif",
+ "--overwrite",
+ "--output", path + "/" + file.replace(/\.nef$/i, ".jpg"),
+ path + "/" + file
+ ]);
+
+ ufraw.on('close', function(code) {
+ if (code != 0) {
+ return reject("UFRAW returned an error");
+ }
+ console.log("UFRAW for " + path + "/" + file + ": " + code);
+ return resolve();
+ fs.rename(path + "/" + file, path + "/raw/" + path, function(err) {
+ if (err) {
+ console.error("Unable to move RAW file: " + path + "/" + file);
+ return reject(err);
+ } else {
+ return resolve();
+ }
+ });
+ });
+ });
+ });
+}
+
+function moveCorrupt(path, file) {
+ path = picturesPath + path;
+ console.warn("Moving corrupt file '" + file + "' to " + path + "/corrupt");
+
+ return mkdirPromise(path + "/corrupt").then(function() {
+ return new Promise(function(resolve, reject) {
+ fs.rename(path + "/" + file, path + "/corrupt/" + file, function(err) {
+ if (err) {
+ console.error("Unable to move corrupt file: " + path + "/" + file);
+ return reject(err);
+ } else {
+ return resolve();
+ }
+ });
+ });
+ });
+}
+
function triggerWatcher() {
setTimeout(triggerWatcher, 1000);
if (!processRunning && processQueue.length) {
- let lastMessage = moment(), toProcess = processQueue.length;
+ let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0);
processRunning = true;
- return Promise.map(processQueue, function(entry) {
- var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3],
- src = picturesPath + path + "/" + file,
- dst = picturesPath + path + "/thumbs/" + file,
- image = sharp(src);
+ return Promise.map(processing, function(entry) {
+ var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3];
+
// console.log("Processing " + src);
- return image.metadata().then(function(metadata) {
- if (metadata.exif) {
- metadata.exif = exif(metadata.exif);
- delete metadata.exif.thumbnail;
- delete metadata.exif.image;
- for (var key in metadata.exif.exif) {
- if (Buffer.isBuffer(metadata.exif.exif[key])) {
- metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
+
+ let tmp = Promise.resolve(file);
+ /* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */
+ if (/\.nef$/i.exec(file)) {
+ tmp = mkdirPromise(picturesPath + path + "/raw").then(function() {
+ return convertNefToJpg(path, file);
+ }).then(function() {
+ return file.replace(/\.nef$/i, ".jpg"); /* We converted from NEF => JPG */
+ });
+ }
+
+ return tmp.then(function(file) {
+ var src = picturesPath + path + "/" + file,
+ dst = picturesPath + path + "/thumbs/" + file,
+ image = sharp(src);
+
+ return image.metadata().then(function(metadata) {
+ if (metadata.exif) {
+ metadata.exif = exif(metadata.exif);
+ delete metadata.exif.thumbnail;
+ delete metadata.exif.image;
+ for (var key in metadata.exif.exif) {
+ if (Buffer.isBuffer(metadata.exif.exif[key])) {
+ metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
+ }
}
}
- }
- let replacements = {
- albumId: albumId,
- path: path,
- filename: file,
- width: metadata.width,
- height: metadata.height,
- added: moment().format().replace(/T.*/, "")
- };
+ let replacements = {
+ albumId: albumId,
+ path: path,
+ filename: file,
+ width: metadata.width,
+ height: metadata.height,
+ added: moment().format().replace(/T.*/, "")
+ };
- if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
- metadata.exif.exif.DateTimeOriginal.setHours(0, 0, 0, 0);
- metadata.exif.exif.DateTimeOriginal = metadata.exif.exif.DateTimeOriginal.toISOString().replace(/T.*/, "");
- replacements.taken = moment(metadata.exif.exif.DateTimeOriginal, "YYYY-MM-DD").format().replace(/T.*/, "");
- replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format().replace(/T.*/, "");
+ if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
+ metadata.exif.exif.DateTimeOriginal.setHours(0, 0, 0, 0);
+ metadata.exif.exif.DateTimeOriginal = metadata.exif.exif.DateTimeOriginal.toISOString().replace(/T.*/, "");
+ replacements.taken = moment(metadata.exif.exif.DateTimeOriginal, "YYYY-MM-DD").format().replace(/T.*/, "");
+ replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format().replace(/T.*/, "");
- if (replacements.taken == "Invalid date") {
- console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
- replacements.taken = replacements.modified = replacements.added;
- }
- } else {
- let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = replacements.added;
- let match = file.match(patterns);
- if (match) {
- date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
- if (date == "Invalid date") {
- date = moment(created).format();
+ if (replacements.taken == "Invalid date" || replacements.taken == "1899-11-30") {
+ console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
+ replacements.taken = replacements.modified = moment(created).format();
}
} else {
- date = moment(created).format();
- }
- replacements.taken = replacements.modified = date;
- }
-
- return image.resize(256, 256).toFile(dst).then(function() {
- return photoDB.sequelize.query("INSERT INTO photos " +
- "SET albumId=:albumId,path=:path,filename=:filename,added=DATE(:added),modified=DATE(:modified),taken=DATE(:taken),width=:width,height=:height", {
- replacements: replacements
- }).then(function() {
- toProcess--;
- if (moment().add(-5, 'seconds') > lastMessage) {
- console.log("Items to be processed: " + toProcess);
- lastMessage = moment();
+ let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = moment(created).format();
+ let match = file.match(patterns);
+ if (match) {
+ date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
+ if (date == "Invalid date") {
+ date = moment(created).format();
+ }
+ } else {
+ date = moment(created).format();
}
+ replacements.taken = replacements.modified = date;
+ }
+
+ return image.resize(256, 256).toFile(dst).then(function() {
+ return photoDB.sequelize.query("INSERT INTO photos " +
+ "SET albumId=:albumId,path=:path,filename=:filename,added=DATE(:added),modified=DATE(:modified),taken=DATE(:taken),width=:width,height=:height", {
+ replacements: replacements
+ }).then(function() {
+ toProcess--;
+ if (moment().add(-5, 'seconds') > lastMessage) {
+ console.log("Items to be processed: " + toProcess);
+ lastMessage = moment();
+ }
+ });
+ }).catch(function(error) {
+ console.error("Error resizing image, writing to disc, or updating DB: " + src, error);
+ throw error;
});
}).catch(function(error) {
- console.log("Error resizing or writing " + src, error);
- return Promise.Reject();
+ console.error("Error reading image " + src + ": ", error);
+ return moveCorrupt(path, file).then(function() {
+
+ /* If the original file was not a NEF, we are done... */
+ if (!/\.nef$/i.exec(entry[1])) {
+ return;
+ }
+
+ /* ... otherwise, attempt to re-convert the NEF->JPG and then resize again */
+ for (var i = 0; i < triedClean.length; i++) {
+ if (triedClean[i] == path + "/" + file) {
+ /* Move the NEF to /corrupt as well so it isn't re-checked again and again... */
+ // return moveCorrupt(path, entry[1]);
+
+ console.error("Already attempted to convert NEF to JPG: " + path + "/" + file);
+ return;
+ }
+ }
+
+ console.warn("Adding " + path + "/" + file + " back onto processing queue.");
+ triedClean.push(path + "/" + file);
+ processQueue.push([ path, file, created, albumId ]);
+ });
});
});
}, {
concurrency: 1
+ }).then(function() {
+ console.log("Completed processing queue.");
});
}
}
@@ -219,7 +355,7 @@ function triggerWatcher() {
module.exports = {
scan: function (db) {
photoDB = db;
- return scanDir(0, picturesPath).then(function() {
+ return scanDir(null, picturesPath).then(function() {
triggerWatcher();
});
}