From e8adf6b8209dd17837e8e9b3bbf7fcf1be7779bc Mon Sep 17 00:00:00 2001
From: James Ketrenos <james_git@ketrenos.com>
Date: Thu, 23 Aug 2018 20:16:12 -0700
Subject: [PATCH] Working; trying sqlite

Signed-off-by: James Ketrenos <james_git@ketrenos.com>
---
 README.md                                 |   2 +
 frontend/src/ketr-photos/ketr-photos.html |  18 +-
 package.json                              |   3 +-
 server/scanner.js                         | 300 ++++++++++++++++------
 4 files changed, 232 insertions(+), 91 deletions(-)

diff --git a/README.md b/README.md
index 494b145..44d162a 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,5 @@
+NEF processing uses ufraw-batch
+
 ### Create `photos` user for DB
 
 You will need to know the root password for your mariadb installation. If you
diff --git a/frontend/src/ketr-photos/ketr-photos.html b/frontend/src/ketr-photos/ketr-photos.html
index b9e157e..ed10739 100755
--- a/frontend/src/ketr-photos/ketr-photos.html
+++ b/frontend/src/ketr-photos/ketr-photos.html
@@ -106,14 +106,16 @@
 
     <photo-thumbnail id="placeholder"></photo-thumbnail>
 
-    <app-header-layout fullbleed has-scrolling-region>
-      <app-header slot="header">
-        <paper-spinner active$="[[loading]]" class="thin"></paper-spinner>
-        <paper-radio-group selected="{{order}}">
-          <paper-radio-button name="by-date">By date</paper-radio-button>
-          <paper-radio-button name="by-album">By album</paper-radio-button>
-        </paper-radio-group>
-        <div>[[path]]</div>
+    <app-header-layout reveals>
+      <app-header fixed>
+        <div>
+          <paper-spinner active$="[[loading]]" class="thin"></paper-spinner>
+          <paper-radio-group selected="{{order}}">
+            <paper-radio-button name="by-date">By date</paper-radio-button>
+            <paper-radio-button name="by-album">By album</paper-radio-button>
+          </paper-radio-group>
+          <div>[[path]]</div>
+        </div>
       </app-header>
       <div id="content">
         <div id="thumbnails" class="thumbnails layout horizontal wrap"></div>
diff --git a/package.json b/package.json
index 2419cca..93316ff 100644
--- a/package.json
+++ b/package.json
@@ -29,7 +29,8 @@
     "qs": "^6.5.2",
     "sequelize": "^4.28.6",
     "sequelize-mysql": "^1.7.0",
-    "sharp": "^0.20.5"
+    "sharp": "^0.20.5",
+    "sqlite3": "^4.0.2"
   },
   "jshintConfig": {
     "undef": true,
diff --git a/server/scanner.js b/server/scanner.js
index a8bcd6c..897daa2 100644
--- a/server/scanner.js
+++ b/server/scanner.js
@@ -11,22 +11,29 @@ let photoDB = null;
 
 const picturesPath = config.get("picturesPath");
 
-const processQueue = [];
+const processQueue = [], triedClean = [];
 
 function scanDir(parent, path) {
-  let extensions = [ "jpg", "jpeg", "png", "gif" ],
-    re = "\.((" + extensions.join(")|(") + "))$";
-  re = new RegExp(re, "i");
-
-  return photoDB.sequelize.query("SELECT id FROM albums WHERE path=:path AND parentId=:parent", {
-    replacements: {
+  let extensions = [ "jpg", "jpeg", "png", "gif", "nef" ],
+    re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
+    replacements = {
       path: path,
       parent: parent || null
-    },
+    };
+
+  let query = "SELECT id FROM albums WHERE path=:path AND ";
+  if (!parent) {
+    query += "parentId IS NULL";
+  } else {
+    query += "parentId=:parent";
+  }
+
+  return photoDB.sequelize.query(query, {
+    replacements: replacements,
     type: photoDB.sequelize.QueryTypes.SELECT
   }).then(function(results) {
     if (results.length == 0) {
-      console.log("Adding " + path + " under " + parent);
+//      console.log("Adding " + path + " under " + parent, replacements);
       return photoDB.sequelize.query("INSERT INTO albums SET path=:path,parentId=:parent", {
         replacements: {
           path: path,
@@ -40,7 +47,7 @@ function scanDir(parent, path) {
     }
   }).then(function(parent) {
     return new Promise(function(resolve, reject) {
-    //    console.log("Scanning path " + path);
+       console.log("Scanning path " + path + " under parent " + parent);
 
       fs.readdir(path, function(err, files) {
         if (err) {
@@ -57,26 +64,31 @@ function scanDir(parent, path) {
             break;
           }
         }
-
-        let tmp = Promise.resolve();
-
+  
+        let tmp;
         if (!hasThumbs) {
-          tmp = new Promise(function(resolve, reject) {
-            fs.mkdir(path + "/thumbs", function(err) {
-              if (err) {
-                return reject("Unable to create " + paths + "/thumbs");
-              }
-              return resolve();
-            });
-          });
+          tmp = mkdirPromise(path + "/thumbs");
+        } else {
+          tmp = Promise.resolve();
         }
 
+        /* Remove 'thumbs' and 'raw' directories from being processed */
+        files = files.filter(function(file) {
+          for (var i = 0; i < files.length; i++) {
+            /* If this file has an original NEF on the system, don't add the JPG to the
+             * DB */
+            if (/\.nef$/i.exec(files[i]) && file == files[i].replace(/\.nef$/i, ".jpg")) {
+              console.log("Skipping DB duplicate for " + files[i]);
+              return false;
+            }
+          }
+
+          return file != "raw" && file != "thumbs" && file != ".git" && file != "corrupt";
+        });
+
         return tmp.then(function() {
           return Promise.map(files, function(file) {
             let filepath = path + "/" + file;
-            if (file == "thumbs") {
-              return resolve(true);
-            }
 
             return new Promise(function(resolve, reject) {
               fs.stat(filepath, function(err, stats) {
@@ -91,14 +103,15 @@ function scanDir(parent, path) {
                   });
                 }
 
-                /* stats.isFile()  */
+                /* Check file extensions */
                 if (!re.exec(file)) {
                   return resolve(true);
                 }
           
+
                 const replacements = {
                   path: path.slice(picturesPath.length),
-                  filename: file
+                  filename: file.replace(/\.nef$/i, ".jpg") /* We will be converting from NEF => JPG */
                 };
 
                 return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", {
@@ -137,81 +150,204 @@ const { spawn } = require('child_process');
 
 const sharp = require("sharp"), exif = require("exif-reader");
 
+function mkdirPromise(path) {
+  return new Promise(function(resolve, reject) {
+    fs.stat(path, function(err, stats) {
+      if (err && err.code != 'ENOENT') {
+        console.warn("Could not stat " + path + "/raw");
+        return reject();
+      }
+
+      if (!err) {
+        return resolve();
+      }
+
+      fs.mkdir(path, function(err) {
+        if (err) {
+          return reject("Unable to create " + path, err);
+        }
+        return resolve();
+      });
+    });
+  });
+}
+
+function convertNefToJpg(path, file) {
+  console.log("Converting " + path + "/" + file);
+
+  path = picturesPath + path;
+  return new Promise(function(resolve, reject) {
+    fs.stat(path + "/" + file.replace(/\.nef$/i, ".jpg"), function(err, stats) {
+      if (!err) {
+        console.log("Skipping already converted file: " + file);
+        return resolve();
+      }
+
+      const ufraw = spawn("ufraw-batch", [
+        "--silent",
+        "--wb=camera",
+        "--rotate=camera",
+        "--out-type=jpg",
+        "--compression=90",
+        "--exif",
+        "--overwrite",
+        "--output", path + "/" + file.replace(/\.nef$/i, ".jpg"),
+        path + "/" + file
+      ]);
+
+      ufraw.on('close', function(code) {
+        if (code != 0) {
+          return reject("UFRAW returned an error");
+        }
+        console.log("UFRAW for " + path + "/" + file + ": " + code);
+        return resolve();
+        fs.rename(path + "/" + file, path + "/raw/" + path, function(err) {
+          if (err) {
+            console.error("Unable to move RAW file: " + path + "/" + file);
+            return reject(err);
+          } else {
+            return resolve();
+          }
+        });
+      });
+    });
+  });
+}
+
+function moveCorrupt(path, file) {
+  path = picturesPath + path;
+  console.warn("Moving corrupt file '" + file + "' to " + path + "/corrupt");
+
+  return mkdirPromise(path + "/corrupt").then(function() {
+    return new Promise(function(resolve, reject) {
+      fs.rename(path + "/" + file, path + "/corrupt/" + file, function(err) {
+        if (err) {
+          console.error("Unable to move corrupt file: " + path + "/" + file);
+          return reject(err);
+        } else {
+          return resolve();
+        }
+      });
+    });
+  });
+}
+
 function triggerWatcher() {
   setTimeout(triggerWatcher, 1000);
   
   if (!processRunning && processQueue.length) {
-    let lastMessage = moment(), toProcess = processQueue.length;
+    let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0);
     processRunning = true;
-    return Promise.map(processQueue, function(entry) {
-      var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3],
-        src = picturesPath + path + "/" + file,
-        dst = picturesPath + path + "/thumbs/" + file,
-        image = sharp(src);
+    return Promise.map(processing, function(entry) {
+      var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3];
+        
 //      console.log("Processing " + src);
-      return image.metadata().then(function(metadata) {
-        if (metadata.exif) {
-          metadata.exif = exif(metadata.exif);
-          delete metadata.exif.thumbnail;
-          delete metadata.exif.image;
-          for (var key in metadata.exif.exif) {
-            if (Buffer.isBuffer(metadata.exif.exif[key])) {
-              metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
+
+      let tmp = Promise.resolve(file);
+      /* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */
+      if (/\.nef$/i.exec(file)) {
+        tmp = mkdirPromise(picturesPath + path + "/raw").then(function() {
+          return convertNefToJpg(path, file);
+        }).then(function() {
+          return file.replace(/\.nef$/i, ".jpg"); /* We converted from NEF => JPG */
+        });
+      }
+
+      return tmp.then(function(file) {
+        var src = picturesPath + path + "/" + file,
+          dst = picturesPath + path + "/thumbs/" + file,
+          image = sharp(src);
+
+        return image.metadata().then(function(metadata) {
+          if (metadata.exif) {
+            metadata.exif = exif(metadata.exif);
+            delete metadata.exif.thumbnail;
+            delete metadata.exif.image;
+            for (var key in metadata.exif.exif) {
+              if (Buffer.isBuffer(metadata.exif.exif[key])) {
+                metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
+              }
             }
           }
-        }
 
-        let replacements = {
-          albumId: albumId,
-          path: path,
-          filename: file,
-          width: metadata.width,
-          height: metadata.height,
-          added: moment().format().replace(/T.*/, "")
-        };
+          let replacements = {
+            albumId: albumId,
+            path: path,
+            filename: file,
+            width: metadata.width,
+            height: metadata.height,
+            added: moment().format().replace(/T.*/, "")
+          };
 
-        if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
-          metadata.exif.exif.DateTimeOriginal.setHours(0, 0, 0, 0);
-          metadata.exif.exif.DateTimeOriginal = metadata.exif.exif.DateTimeOriginal.toISOString().replace(/T.*/, "");
-          replacements.taken = moment(metadata.exif.exif.DateTimeOriginal, "YYYY-MM-DD").format().replace(/T.*/, "");
-          replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format().replace(/T.*/, "");
+          if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
+            metadata.exif.exif.DateTimeOriginal.setHours(0, 0, 0, 0);
+            metadata.exif.exif.DateTimeOriginal = metadata.exif.exif.DateTimeOriginal.toISOString().replace(/T.*/, "");
+            replacements.taken = moment(metadata.exif.exif.DateTimeOriginal, "YYYY-MM-DD").format().replace(/T.*/, "");
+            replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format().replace(/T.*/, "");
 
-          if (replacements.taken == "Invalid date") {
-            console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
-            replacements.taken = replacements.modified = replacements.added;
-          }
-        } else {
-          let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = replacements.added;
-          let match = file.match(patterns);
-          if (match) {
-            date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
-            if (date == "Invalid date") {
-              date = moment(created).format();
+            if (replacements.taken == "Invalid date" || replacements.taken == "1899-11-30") {
+              console.log("Invalid EXIF date information: ", JSON.stringify(metadata.exif.exif));
+              replacements.taken = replacements.modified = moment(created).format();
             }
           } else {
-            date = moment(created).format();
-          }
-          replacements.taken = replacements.modified = date;
-        }
-
-        return image.resize(256, 256).toFile(dst).then(function() {
-          return photoDB.sequelize.query("INSERT INTO photos " +
-            "SET albumId=:albumId,path=:path,filename=:filename,added=DATE(:added),modified=DATE(:modified),taken=DATE(:taken),width=:width,height=:height", {
-            replacements: replacements
-          }).then(function() {
-            toProcess--;
-            if (moment().add(-5, 'seconds') > lastMessage) {
-              console.log("Items to be processed: "  + toProcess);
-              lastMessage = moment();
+            let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = moment(created).format();
+            let match = file.match(patterns);
+            if (match) {
+              date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
+              if (date == "Invalid date") {
+                date = moment(created).format();
+              }
+            } else {
+              date = moment(created).format();
             }
+            replacements.taken = replacements.modified = date;
+          }
+
+          return image.resize(256, 256).toFile(dst).then(function() {
+            return photoDB.sequelize.query("INSERT INTO photos " +
+              "SET albumId=:albumId,path=:path,filename=:filename,added=DATE(:added),modified=DATE(:modified),taken=DATE(:taken),width=:width,height=:height", {
+              replacements: replacements
+            }).then(function() {
+              toProcess--;
+              if (moment().add(-5, 'seconds') > lastMessage) {
+                console.log("Items to be processed: "  + toProcess);
+                lastMessage = moment();
+              }
+            });
+          }).catch(function(error) {
+            console.error("Error resizing image, writing to disc, or updating DB: " + src, error);
+            throw error;
           });
         }).catch(function(error) {
-          console.log("Error resizing or writing " + src, error);
-          return Promise.Reject();
+          console.error("Error reading image " + src + ": ", error);
+          return moveCorrupt(path, file).then(function() {
+
+            /* If the original file was not a NEF, we are done... */
+            if (!/\.nef$/i.exec(entry[1])) {
+              return;
+            }
+
+            /* ... otherwise, attempt to re-convert the NEF->JPG and then resize again */
+            for (var i = 0; i < triedClean.length; i++) {
+              if (triedClean[i] == path + "/" + file) {
+                /* Move the NEF to /corrupt as well so it isn't re-checked again and again... */
+                // return moveCorrupt(path, entry[1]);
+
+                console.error("Already attempted to convert NEF to JPG: " + path + "/" + file);
+                return;
+              }
+            }
+
+            console.warn("Adding " + path + "/" + file + " back onto processing queue.");
+            triedClean.push(path + "/" + file);
+            processQueue.push([ path, file, created, albumId ]);
+          });
         });
       });
     }, {
       concurrency: 1
+    }).then(function() {
+      console.log("Completed processing queue.");
     });
   }
 }
@@ -219,7 +355,7 @@ function triggerWatcher() {
 module.exports = {
   scan: function (db) {
     photoDB = db;
-    return scanDir(0, picturesPath).then(function() {
+    return scanDir(null, picturesPath).then(function() {
       triggerWatcher();
     });
   }