Working; trying sqlite
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
parent
5f9fbe21fb
commit
e8adf6b820
@ -1,3 +1,5 @@
|
||||
NEF processing uses ufraw-batch
|
||||
|
||||
### Create `photos` user for DB
|
||||
|
||||
You will need to know the root password for your mariadb installation. If you
|
||||
|
||||
@ -106,14 +106,16 @@
|
||||
|
||||
<photo-thumbnail id="placeholder"></photo-thumbnail>
|
||||
|
||||
<app-header-layout fullbleed has-scrolling-region>
|
||||
<app-header slot="header">
|
||||
<paper-spinner active$="[[loading]]" class="thin"></paper-spinner>
|
||||
<paper-radio-group selected="{{order}}">
|
||||
<paper-radio-button name="by-date">By date</paper-radio-button>
|
||||
<paper-radio-button name="by-album">By album</paper-radio-button>
|
||||
</paper-radio-group>
|
||||
<div>[[path]]</div>
|
||||
<app-header-layout reveals>
|
||||
<app-header fixed>
|
||||
<div>
|
||||
<paper-spinner active$="[[loading]]" class="thin"></paper-spinner>
|
||||
<paper-radio-group selected="{{order}}">
|
||||
<paper-radio-button name="by-date">By date</paper-radio-button>
|
||||
<paper-radio-button name="by-album">By album</paper-radio-button>
|
||||
</paper-radio-group>
|
||||
<div>[[path]]</div>
|
||||
</div>
|
||||
</app-header>
|
||||
<div id="content">
|
||||
<div id="thumbnails" class="thumbnails layout horizontal wrap"></div>
|
||||
|
||||
@ -29,7 +29,8 @@
|
||||
"qs": "^6.5.2",
|
||||
"sequelize": "^4.28.6",
|
||||
"sequelize-mysql": "^1.7.0",
|
||||
"sharp": "^0.20.5"
|
||||
"sharp": "^0.20.5",
|
||||
"sqlite3": "^4.0.2"
|
||||
},
|
||||
"jshintConfig": {
|
||||
"undef": true,
|
||||
|
||||
@ -11,22 +11,29 @@ let photoDB = null;
|
||||
|
||||
const picturesPath = config.get("picturesPath");
|
||||
|
||||
const processQueue = [];
|
||||
const processQueue = [], triedClean = [];
|
||||
|
||||
function scanDir(parent, path) {
|
||||
let extensions = [ "jpg", "jpeg", "png", "gif" ],
|
||||
re = "\.((" + extensions.join(")|(") + "))$";
|
||||
re = new RegExp(re, "i");
|
||||
|
||||
return photoDB.sequelize.query("SELECT id FROM albums WHERE path=:path AND parentId=:parent", {
|
||||
replacements: {
|
||||
let extensions = [ "jpg", "jpeg", "png", "gif", "nef" ],
|
||||
re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
|
||||
replacements = {
|
||||
path: path,
|
||||
parent: parent || null
|
||||
},
|
||||
};
|
||||
|
||||
let query = "SELECT id FROM albums WHERE path=:path AND ";
|
||||
if (!parent) {
|
||||
query += "parentId IS NULL";
|
||||
} else {
|
||||
query += "parentId=:parent";
|
||||
}
|
||||
|
||||
return photoDB.sequelize.query(query, {
|
||||
replacements: replacements,
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}).then(function(results) {
|
||||
if (results.length == 0) {
|
||||
console.log("Adding " + path + " under " + parent);
|
||||
// console.log("Adding " + path + " under " + parent, replacements);
|
||||
return photoDB.sequelize.query("INSERT INTO albums SET path=:path,parentId=:parent", {
|
||||
replacements: {
|
||||
path: path,
|
||||
@ -40,7 +47,7 @@ function scanDir(parent, path) {
|
||||
}
|
||||
}).then(function(parent) {
|
||||
return new Promise(function(resolve, reject) {
|
||||
// console.log("Scanning path " + path);
|
||||
console.log("Scanning path " + path + " under parent " + parent);
|
||||
|
||||
fs.readdir(path, function(err, files) {
|
||||
if (err) {
|
||||
@ -57,26 +64,31 @@ function scanDir(parent, path) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let tmp = Promise.resolve();
|
||||
|
||||
|
||||
let tmp;
|
||||
if (!hasThumbs) {
|
||||
tmp = new Promise(function(resolve, reject) {
|
||||
fs.mkdir(path + "/thumbs", function(err) {
|
||||
if (err) {
|
||||
return reject("Unable to create " + paths + "/thumbs");
|
||||
}
|
||||
return resolve();
|
||||
});
|
||||
});
|
||||
tmp = mkdirPromise(path + "/thumbs");
|
||||
} else {
|
||||
tmp = Promise.resolve();
|
||||
}
|
||||
|
||||
/* Remove 'thumbs' and 'raw' directories from being processed */
|
||||
files = files.filter(function(file) {
|
||||
for (var i = 0; i < files.length; i++) {
|
||||
/* If this file has an original NEF on the system, don't add the JPG to the
|
||||
* DB */
|
||||
if (/\.nef$/i.exec(files[i]) && file == files[i].replace(/\.nef$/i, ".jpg")) {
|
||||
console.log("Skipping DB duplicate for " + files[i]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return file != "raw" && file != "thumbs" && file != ".git" && file != "corrupt";
|
||||
});
|
||||
|
||||
return tmp.then(function() {
|
||||
return Promise.map(files, function(file) {
|
||||
let filepath = path + "/" + file;
|
||||
if (file == "thumbs") {
|
||||
return resolve(true);
|
||||
}
|
||||
|
||||
return new Promise(function(resolve, reject) {
|
||||
fs.stat(filepath, function(err, stats) {
|
||||
@ -91,14 +103,15 @@ function scanDir(parent, path) {
|
||||
});
|
||||
}
|
||||
|
||||
/* stats.isFile() */
|
||||
/* Check file extensions */
|
||||
if (!re.exec(file)) {
|
||||
return resolve(true);
|
||||
}
|
||||
|
||||
|
||||
const replacements = {
|
||||
path: path.slice(picturesPath.length),
|
||||
filename: file
|
||||
filename: file.replace(/\.nef$/i, ".jpg") /* We will be converting from NEF => JPG */
|
||||
};
|
||||
|
||||
return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", {
|
||||
@ -137,81 +150,204 @@ const { spawn } = require('child_process');
|
||||
|
||||
const sharp = require("sharp"), exif = require("exif-reader");
|
||||
|
||||
function mkdirPromise(path) {
|
||||
return new Promise(function(resolve, reject) {
|
||||
fs.stat(path, function(err, stats) {
|
||||
if (err && err.code != 'ENOENT') {
|
||||
console.warn("Could not stat " + path + "/raw");
|
||||
return reject();
|
||||
}
|
||||
|
||||
if (!err) {
|
||||
return resolve();
|
||||
}
|
||||
|
||||
fs.mkdir(path, function(err) {
|
||||
if (err) {
|
||||
return reject("Unable to create " + path, err);
|
||||
}
|
||||
return resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function convertNefToJpg(path, file) {
|
||||
console.log("Converting " + path + "/" + file);
|
||||
|
||||
path = picturesPath + path;
|
||||
return new Promise(function(resolve, reject) {
|
||||
fs.stat(path + "/" + file.replace(/\.nef$/i, ".jpg"), function(err, stats) {
|
||||
if (!err) {
|
||||
console.log("Skipping already converted file: " + file);
|
||||
return resolve();
|
||||
}
|
||||
|
||||
const ufraw = spawn("ufraw-batch", [
|
||||
"--silent",
|
||||
"--wb=camera",
|
||||
"--rotate=camera",
|
||||
"--out-type=jpg",
|
||||
"--compression=90",
|
||||
"--exif",
|
||||
"--overwrite",
|
||||
"--output", path + "/" + file.replace(/\.nef$/i, ".jpg"),
|
||||
path + "/" + file
|
||||
]);
|
||||
|
||||
ufraw.on('close', function(code) {
|
||||
if (code != 0) {
|
||||
return reject("UFRAW returned an error");
|
||||
}
|
||||
console.log("UFRAW for " + path + "/" + file + ": " + code);
|
||||
return resolve();
|
||||
fs.rename(path + "/" + file, path + "/raw/" + path, function(err) {
|
||||
if (err) {
|
||||
console.error("Unable to move RAW file: " + path + "/" + file);
|
||||
return reject(err);
|
||||
} else {
|
||||
return resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function moveCorrupt(path, file) {
|
||||
path = picturesPath + path;
|
||||
console.warn("Moving corrupt file '" + file + "' to " + path + "/corrupt");
|
||||
|
||||
return mkdirPromise(path + "/corrupt").then(function() {
|
||||
return new Promise(function(resolve, reject) {
|
||||
fs.rename(path + "/" + file, path + "/corrupt/" + file, function(err) {
|
||||
if (err) {
|
||||
console.error("Unable to move corrupt file: " + path + "/" + file);
|
||||
return reject(err);
|
||||
} else {
|
||||
return resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function triggerWatcher() {
|
||||
setTimeout(triggerWatcher, 1000);
|
||||
|
||||
if (!processRunning && processQueue.length) {
|
||||
let lastMessage = moment(), toProcess = processQueue.length;
|
||||
let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0);
|
||||
processRunning = true;
|
||||
return Promise.map(processQueue, function(entry) {
|
||||
var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3],
|
||||
src = picturesPath + path + "/" + file,
|
||||
dst = picturesPath + path + "/thumbs/" + file,
|
||||
image = sharp(src);
|
||||
return Promise.map(processing, function(entry) {
|
||||
var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3];
|
||||
|
||||
// console.log("Processing " + src);
|
||||
return image.metadata().then(function(metadata) {
|
||||
if (metadata.exif) {
|
||||
metadata.exif = exif(metadata.exif);
|
||||
delete metadata.exif.thumbnail;
|
||||
delete metadata.exif.image;
|
||||
for (var key in metadata.exif.exif) {
|
||||
if (Buffer.isBuffer(metadata.exif.exif[key])) {
|
||||
metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
|
||||
|
||||
let tmp = Promise.resolve(file);
|
||||
/* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */
|
||||
if (/\.nef$/i.exec(file)) {
|
||||
tmp = mkdirPromise(picturesPath + path + "/raw").then(function() {
|
||||
return convertNefToJpg(path, file);
|
||||
}).then(function() {
|
||||
return file.replace(/\.nef$/i, ".jpg"); /* We converted from NEF => JPG */
|
||||
});
|
||||
}
|
||||
|
||||
return tmp.then(function(file) {
|
||||
var src = picturesPath + path + "/" + file,
|
||||
dst = picturesPath + path + "/thumbs/" + file,
|
||||
image = sharp(src);
|
||||
|
||||
return image.metadata().then(function(metadata) {
|
||||
if (metadata.exif) {
|
||||
metadata.exif = exif(metadata.exif);
|
||||
delete metadata.exif.thumbnail;
|
||||
delete metadata.exif.image;
|
||||
for (var key in metadata.exif.exif) {
|
||||
if (Buffer.isBuffer(metadata.exif.exif[key])) {
|
||||
metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let replacements = {
|
||||
albumId: albumId,
|
||||
path: path,
|
||||
filename: file,
|
||||
width: metadata.width,
|
||||
height: metadata.height,
|
||||
added: moment().format().replace(/T.*/, "")
|
||||
};
|
||||
let replacements = {
|
||||
albumId: albumId,
|
||||
path: path,
|
||||
filename: file,
|
||||
width: metadata.width,
|
||||
height: metadata.height,
|
||||
added: moment().format().replace(/T.*/, "")
|
||||
};
|
||||
|
||||
if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
|
||||
metadata.exif.exif.DateTimeOriginal.setHours(0, 0, 0, 0);
|
||||
metadata.exif.exif.DateTimeOriginal = metadata.exif.exif.DateTimeOriginal.toISOString().replace(/T.*/, "");
|
||||
replacements.taken = moment(metadata.exif.exif.DateTimeOriginal, "YYYY-MM-DD").format().replace(/T.*/, "");
|
||||
replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format().replace(/T.*/, "");
|
||||
if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
|
||||
metadata.exif.exif.DateTimeOriginal.setHours(0, 0, 0, 0);
|
||||
metadata.exif.exif.DateTimeOriginal = metadata.exif.exif.DateTimeOriginal.toISOString().replace(/T.*/, "");
|
||||
replacements.taken = moment(metadata.exif.exif.DateTimeOriginal, "YYYY-MM-DD").format().replace(/T.*/, "");
|
||||
replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format().replace(/T.*/, "");
|
||||
|
||||
if (replacements.taken == "Invalid date") {
|
||||
console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
|
||||
replacements.taken = replacements.modified = replacements.added;
|
||||
}
|
||||
} else {
|
||||
let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = replacements.added;
|
||||
let match = file.match(patterns);
|
||||
if (match) {
|
||||
date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
|
||||
if (date == "Invalid date") {
|
||||
date = moment(created).format();
|
||||
if (replacements.taken == "Invalid date" || replacements.taken == "1899-11-30") {
|
||||
console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
|
||||
replacements.taken = replacements.modified = moment(created).format();
|
||||
}
|
||||
} else {
|
||||
date = moment(created).format();
|
||||
}
|
||||
replacements.taken = replacements.modified = date;
|
||||
}
|
||||
|
||||
return image.resize(256, 256).toFile(dst).then(function() {
|
||||
return photoDB.sequelize.query("INSERT INTO photos " +
|
||||
"SET albumId=:albumId,path=:path,filename=:filename,added=DATE(:added),modified=DATE(:modified),taken=DATE(:taken),width=:width,height=:height", {
|
||||
replacements: replacements
|
||||
}).then(function() {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
console.log("Items to be processed: " + toProcess);
|
||||
lastMessage = moment();
|
||||
let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = moment(created).format();
|
||||
let match = file.match(patterns);
|
||||
if (match) {
|
||||
date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
|
||||
if (date == "Invalid date") {
|
||||
date = moment(created).format();
|
||||
}
|
||||
} else {
|
||||
date = moment(created).format();
|
||||
}
|
||||
replacements.taken = replacements.modified = date;
|
||||
}
|
||||
|
||||
return image.resize(256, 256).toFile(dst).then(function() {
|
||||
return photoDB.sequelize.query("INSERT INTO photos " +
|
||||
"SET albumId=:albumId,path=:path,filename=:filename,added=DATE(:added),modified=DATE(:modified),taken=DATE(:taken),width=:width,height=:height", {
|
||||
replacements: replacements
|
||||
}).then(function() {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
console.log("Items to be processed: " + toProcess);
|
||||
lastMessage = moment();
|
||||
}
|
||||
});
|
||||
}).catch(function(error) {
|
||||
console.error("Error resizing image, writing to disc, or updating DB: " + src, error);
|
||||
throw error;
|
||||
});
|
||||
}).catch(function(error) {
|
||||
console.log("Error resizing or writing " + src, error);
|
||||
return Promise.Reject();
|
||||
console.error("Error reading image " + src + ": ", error);
|
||||
return moveCorrupt(path, file).then(function() {
|
||||
|
||||
/* If the original file was not a NEF, we are done... */
|
||||
if (!/\.nef$/i.exec(entry[1])) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* ... otherwise, attempt to re-convert the NEF->JPG and then resize again */
|
||||
for (var i = 0; i < triedClean.length; i++) {
|
||||
if (triedClean[i] == path + "/" + file) {
|
||||
/* Move the NEF to /corrupt as well so it isn't re-checked again and again... */
|
||||
// return moveCorrupt(path, entry[1]);
|
||||
|
||||
console.error("Already attempted to convert NEF to JPG: " + path + "/" + file);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
console.warn("Adding " + path + "/" + file + " back onto processing queue.");
|
||||
triedClean.push(path + "/" + file);
|
||||
processQueue.push([ path, file, created, albumId ]);
|
||||
});
|
||||
});
|
||||
});
|
||||
}, {
|
||||
concurrency: 1
|
||||
}).then(function() {
|
||||
console.log("Completed processing queue.");
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -219,7 +355,7 @@ function triggerWatcher() {
|
||||
module.exports = {
|
||||
scan: function (db) {
|
||||
photoDB = db;
|
||||
return scanDir(0, picturesPath).then(function() {
|
||||
return scanDir(null, picturesPath).then(function() {
|
||||
triggerWatcher();
|
||||
});
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user