Reworking scan

Signed-off-by: James Ketrenos <james_git@ketrenos.com>

frontend/src/ketr-photos/ketr-photos.html

@@ -32,6 +32,7 @@
 
 <link rel="import" href="../../elements/photo-lightbox.html">
 <link rel="import" href="../../elements/photo-thumbnail.html">
+<link rel="import" href="../../elements/pan-line.html">
 
 <script src="fetch.js"></script>
 

server/db/photos.js

@@ -72,6 +72,24 @@ function init() {
       timestamps: false
     });
 
+    const PhotoHash = db.sequelize.define('photohash', {
+      hash: {
+        type: Sequelize.STRING,
+        primaryKey: true,
+        unique: true
+      },
+      photoId: {
+        type: Sequelize.INTEGER,
+        allowNull: true,
+        references: {
+          model: Photo,
+          key: 'id',
+        }
+      }
+    }, {
+      timestamps: false
+    });
+
     return db.sequelize.sync({
       force: false
     }).then(function () {

server/scanner.js

@@ -37,7 +37,7 @@ function removeNewerFile(path, fileA, fileB) {
   });
 }
 
-function scanDir(parent, path) {
+function _scanDir(parent, path) {
   let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
     replacements = {
       path: path.slice(picturesPath.length),
@@ -206,7 +206,6 @@ const mkdir = function (_path) {
   });
 }
 
-
 const exists = function(path) {
   return stat(path).then(function() {
     return true;
@@ -462,12 +461,97 @@ function triggerWatcher() {
     });
   }
 }
+/*******************************************************************************************************/
+function scanDir(parent, path) {
+  let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
+    album = {
+      path: path.slice(picturesPath.length),
+      name: path.replace(/.*\//, "").replace(/_/g, " "),
+      assets: [],
+      parent: parent,
+      allAssetCount: 0,
+      allAlbumCount: 0
+    };
+
+  return new Promise(function(resolve, reject) {
+    fs.readdir(path, function(error, files) {
+      if (error) {
+        console.warn("Could not readdir: " + path);
+        return resolve([]);
+      }
+
+      /* Remove 'thumbs' and 'raw' directories from being processed */
+      files = files.filter(function(file) {
+        for (var i = 0; i < files.length; i++) {
+          /* If this file has an original NEF/ORF on the system, don't add the JPG to the DB */
+          if (rawExtension.exec(files[i]) && file == files[i].replace(rawExtension, ".jpg")) {
+            return false;
+          }
+          
+          /* If there is a different CASE (eg. JPG vs jpg) don't add it, and remove the 'lower case' 
+            * version from disk. */
+          if (file != files[i] && file.toUpperCase() == files[i]) {
+            removeNewerFile(path, file, files[i]);
+            console.log("Duplicate file in " + path + ": ", file, files[i]);
+            return false;
+          }
+        }
+
+        return file != "raw" && file != "thumbs" && file != ".git" && file != "corrupt";
+      });
+
+      return resolve(files);
+    });
+  }).then(function(files) {
+    return Promise.map(files, function(file) {
+      let filepath = path + "/" + file;
+
+      return stat(filepath).then(function(stats) {
+        if (stats.isDirectory()) {
+          return scanDir(album, filepath).then(function(child) {
+            album.allAssetCount += child.allAssetCount;
+            album.allAlbumCount += child.allAlbumCount + 1;
+          }).catch(function(error) {
+            console.warn("Could not scanDir " + filepath + ": " + error);
+          });
+        }
+
+        /* Check file extensions */
+        if (!re.exec(file)) {
+          return;
+        }
+    
+        album.allAssetCount++;
+        album.assets.push({
+          path: path.slice(picturesPath.length),
+          filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
+          stats: stats
+        });
+      });
+    });
+  }).then(function() {
+    return album;
+  });
+}
+
 
 module.exports = {
   scan: function (db) {
     photoDB = db;
-    return scanDir(null, picturesPath).then(function() {
+    /* 1. Scan for all assets which will be managed by the system. readdir
-      triggerWatcher();
+      - returns set of albums and photos.
+     * 2. Compute HASH of the file
+     * 3. Check for HASH in photohash -- skip?
+     * 4. Check for and create thumbs/FILE thumbs/scaled/FILE
+     * 5. If necessary, create JPG from RAW
+     * 6. Check in DB and update last-scanned date
+     * 7. Look up all DB entries with last-scanned date < NOW -- purge?
+     */
+    let now = Date.now();
+    return scanDir(null, picturesPath).then(function(albums) {
+      console.log("Found " + albums.allAssetCount + " assets in " + albums.allAlbumCount + " albums after " + 
+        ((Date.now() - now) / 1000) + "s");
+      /*triggerWatcher();*/
     });
   }
 };