Working; trying sqlite

Signed-off-by: James Ketrenos <james_git@ketrenos.com>

README.md

@@ -1,3 +1,5 @@
+NEF processing uses ufraw-batch
+
 ### Create `photos` user for DB
 
 You will need to know the root password for your mariadb installation. If you

frontend/src/ketr-photos/ketr-photos.html

@@ -106,14 +106,16 @@
 
     <photo-thumbnail id="placeholder"></photo-thumbnail>
 
-    <app-header-layout fullbleed has-scrolling-region>
+    <app-header-layout reveals>
-      <app-header slot="header">
+      <app-header fixed>
+        <div>
           <paper-spinner active$="[[loading]]" class="thin"></paper-spinner>
           <paper-radio-group selected="{{order}}">
             <paper-radio-button name="by-date">By date</paper-radio-button>
             <paper-radio-button name="by-album">By album</paper-radio-button>
           </paper-radio-group>
           <div>[[path]]</div>
+        </div>
       </app-header>
       <div id="content">
         <div id="thumbnails" class="thumbnails layout horizontal wrap"></div>

package.json

@@ -29,7 +29,8 @@
     "qs": "^6.5.2",
     "sequelize": "^4.28.6",
     "sequelize-mysql": "^1.7.0",
-    "sharp": "^0.20.5"
+    "sharp": "^0.20.5",
+    "sqlite3": "^4.0.2"
   },
   "jshintConfig": {
     "undef": true,

server/scanner.js

@@ -11,22 +11,29 @@ let photoDB = null;
 
 const picturesPath = config.get("picturesPath");
 
-const processQueue = [];
+const processQueue = [], triedClean = [];
 
 function scanDir(parent, path) {
-  let extensions = [ "jpg", "jpeg", "png", "gif" ],
+  let extensions = [ "jpg", "jpeg", "png", "gif", "nef" ],
-    re = "\.((" + extensions.join(")|(") + "))$";
+    re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
-  re = new RegExp(re, "i");
+    replacements = {
-
-  return photoDB.sequelize.query("SELECT id FROM albums WHERE path=:path AND parentId=:parent", {
-    replacements: {
       path: path,
       parent: parent || null
-    },
+    };
+
+  let query = "SELECT id FROM albums WHERE path=:path AND ";
+  if (!parent) {
+    query += "parentId IS NULL";
+  } else {
+    query += "parentId=:parent";
+  }
+
+  return photoDB.sequelize.query(query, {
+    replacements: replacements,
     type: photoDB.sequelize.QueryTypes.SELECT
   }).then(function(results) {
     if (results.length == 0) {
-      console.log("Adding " + path + " under " + parent);
+//      console.log("Adding " + path + " under " + parent, replacements);
       return photoDB.sequelize.query("INSERT INTO albums SET path=:path,parentId=:parent", {
         replacements: {
           path: path,
@@ -40,7 +47,7 @@ function scanDir(parent, path) {
     }
   }).then(function(parent) {
     return new Promise(function(resolve, reject) {
-    //    console.log("Scanning path " + path);
+       console.log("Scanning path " + path + " under parent " + parent);
 
       fs.readdir(path, function(err, files) {
         if (err) {
@@ -58,25 +65,30 @@ function scanDir(parent, path) {
           }
         }
   
-        let tmp = Promise.resolve();
+        let tmp;
-
         if (!hasThumbs) {
-          tmp = new Promise(function(resolve, reject) {
+          tmp = mkdirPromise(path + "/thumbs");
-            fs.mkdir(path + "/thumbs", function(err) {
+        } else {
-              if (err) {
+          tmp = Promise.resolve();
-                return reject("Unable to create " + paths + "/thumbs");
         }
-              return resolve();
+
-            });
+        /* Remove 'thumbs' and 'raw' directories from being processed */
-          });
+        files = files.filter(function(file) {
+          for (var i = 0; i < files.length; i++) {
+            /* If this file has an original NEF on the system, don't add the JPG to the
+             * DB */
+            if (/\.nef$/i.exec(files[i]) && file == files[i].replace(/\.nef$/i, ".jpg")) {
+              console.log("Skipping DB duplicate for " + files[i]);
+              return false;
             }
+          }
+
+          return file != "raw" && file != "thumbs" && file != ".git" && file != "corrupt";
+        });
 
         return tmp.then(function() {
           return Promise.map(files, function(file) {
             let filepath = path + "/" + file;
-            if (file == "thumbs") {
-              return resolve(true);
-            }
 
             return new Promise(function(resolve, reject) {
               fs.stat(filepath, function(err, stats) {
@@ -91,14 +103,15 @@ function scanDir(parent, path) {
                   });
                 }
 
-                /* stats.isFile()  */
+                /* Check file extensions */
                 if (!re.exec(file)) {
                   return resolve(true);
                 }
           
+
                 const replacements = {
                   path: path.slice(picturesPath.length),
-                  filename: file
+                  filename: file.replace(/\.nef$/i, ".jpg") /* We will be converting from NEF => JPG */
                 };
 
                 return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", {
@@ -137,18 +150,114 @@ const { spawn } = require('child_process');
 
 const sharp = require("sharp"), exif = require("exif-reader");
 
+function mkdirPromise(path) {
+  return new Promise(function(resolve, reject) {
+    fs.stat(path, function(err, stats) {
+      if (err && err.code != 'ENOENT') {
+        console.warn("Could not stat " + path + "/raw");
+        return reject();
+      }
+
+      if (!err) {
+        return resolve();
+      }
+
+      fs.mkdir(path, function(err) {
+        if (err) {
+          return reject("Unable to create " + path, err);
+        }
+        return resolve();
+      });
+    });
+  });
+}
+
+function convertNefToJpg(path, file) {
+  console.log("Converting " + path + "/" + file);
+
+  path = picturesPath + path;
+  return new Promise(function(resolve, reject) {
+    fs.stat(path + "/" + file.replace(/\.nef$/i, ".jpg"), function(err, stats) {
+      if (!err) {
+        console.log("Skipping already converted file: " + file);
+        return resolve();
+      }
+
+      const ufraw = spawn("ufraw-batch", [
+        "--silent",
+        "--wb=camera",
+        "--rotate=camera",
+        "--out-type=jpg",
+        "--compression=90",
+        "--exif",
+        "--overwrite",
+        "--output", path + "/" + file.replace(/\.nef$/i, ".jpg"),
+        path + "/" + file
+      ]);
+
+      ufraw.on('close', function(code) {
+        if (code != 0) {
+          return reject("UFRAW returned an error");
+        }
+        console.log("UFRAW for " + path + "/" + file + ": " + code);
+        return resolve();
+        fs.rename(path + "/" + file, path + "/raw/" + path, function(err) {
+          if (err) {
+            console.error("Unable to move RAW file: " + path + "/" + file);
+            return reject(err);
+          } else {
+            return resolve();
+          }
+        });
+      });
+    });
+  });
+}
+
+function moveCorrupt(path, file) {
+  path = picturesPath + path;
+  console.warn("Moving corrupt file '" + file + "' to " + path + "/corrupt");
+
+  return mkdirPromise(path + "/corrupt").then(function() {
+    return new Promise(function(resolve, reject) {
+      fs.rename(path + "/" + file, path + "/corrupt/" + file, function(err) {
+        if (err) {
+          console.error("Unable to move corrupt file: " + path + "/" + file);
+          return reject(err);
+        } else {
+          return resolve();
+        }
+      });
+    });
+  });
+}
+
 function triggerWatcher() {
   setTimeout(triggerWatcher, 1000);
   
   if (!processRunning && processQueue.length) {
-    let lastMessage = moment(), toProcess = processQueue.length;
+    let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0);
     processRunning = true;
-    return Promise.map(processQueue, function(entry) {
+    return Promise.map(processing, function(entry) {
-      var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3],
+      var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3];
-        src = picturesPath + path + "/" + file,
+        
+//      console.log("Processing " + src);
+
+      let tmp = Promise.resolve(file);
+      /* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */
+      if (/\.nef$/i.exec(file)) {
+        tmp = mkdirPromise(picturesPath + path + "/raw").then(function() {
+          return convertNefToJpg(path, file);
+        }).then(function() {
+          return file.replace(/\.nef$/i, ".jpg"); /* We converted from NEF => JPG */
+        });
+      }
+
+      return tmp.then(function(file) {
+        var src = picturesPath + path + "/" + file,
           dst = picturesPath + path + "/thumbs/" + file,
           image = sharp(src);
-//      console.log("Processing " + src);
+
         return image.metadata().then(function(metadata) {
           if (metadata.exif) {
             metadata.exif = exif(metadata.exif);
@@ -176,12 +285,12 @@ function triggerWatcher() {
             replacements.taken = moment(metadata.exif.exif.DateTimeOriginal, "YYYY-MM-DD").format().replace(/T.*/, "");
             replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format().replace(/T.*/, "");
 
-          if (replacements.taken == "Invalid date") {
+            if (replacements.taken == "Invalid date" || replacements.taken == "1899-11-30") {
               console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
-            replacements.taken = replacements.modified = replacements.added;
+              replacements.taken = replacements.modified = moment(created).format();
             }
           } else {
-          let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = replacements.added;
+            let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = moment(created).format();
             let match = file.match(patterns);
             if (match) {
               date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
@@ -206,12 +315,39 @@ function triggerWatcher() {
               }
             });
           }).catch(function(error) {
-          console.log("Error resizing or writing " + src, error);
+            console.error("Error resizing image, writing to disc, or updating DB: " + src, error);
-          return Promise.Reject();
+            throw error;
+          });
+        }).catch(function(error) {
+          console.error("Error reading image " + src + ": ", error);
+          return moveCorrupt(path, file).then(function() {
+
+            /* If the original file was not a NEF, we are done... */
+            if (!/\.nef$/i.exec(entry[1])) {
+              return;
+            }
+
+            /* ... otherwise, attempt to re-convert the NEF->JPG and then resize again */
+            for (var i = 0; i < triedClean.length; i++) {
+              if (triedClean[i] == path + "/" + file) {
+                /* Move the NEF to /corrupt as well so it isn't re-checked again and again... */
+                // return moveCorrupt(path, entry[1]);
+
+                console.error("Already attempted to convert NEF to JPG: " + path + "/" + file);
+                return;
+              }
+            }
+
+            console.warn("Adding " + path + "/" + file + " back onto processing queue.");
+            triedClean.push(path + "/" + file);
+            processQueue.push([ path, file, created, albumId ]);
+          });
         });
       });
     }, {
       concurrency: 1
+    }).then(function() {
+      console.log("Completed processing queue.");
     });
   }
 }
@@ -219,7 +355,7 @@ function triggerWatcher() {
 module.exports = {
   scan: function (db) {
     photoDB = db;
-    return scanDir(0, picturesPath).then(function() {
+    return scanDir(null, picturesPath).then(function() {
       triggerWatcher();
     });
   }