From ef74300974de4a877928ea54f6ad5a36a0430b67 Mon Sep 17 00:00:00 2001
From: James Ketrenos <james_git@ketrenos.com>
Date: Mon, 24 Sep 2018 11:16:20 -0700
Subject: [PATCH] Re-scan code working faster

Signed-off-by: James Ketrenos <james_git@ketrenos.com>
---
 server/db/photos.js |  2 +-
 server/scanner.js   | 81 ++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 74 insertions(+), 9 deletions(-)
 mode change 100644 => 100755 server/db/photos.js

diff --git a/server/db/photos.js b/server/db/photos.js
old mode 100644
new mode 100755
index 36f6155..77ca84b
--- a/server/db/photos.js
+++ b/server/db/photos.js
@@ -53,10 +53,10 @@ function init() {
         autoIncrement: true
       },
       name: Sequelize.STRING,
-      path: Sequelize.STRING,
       filename: Sequelize.STRING,
       added: Sequelize.DATE,
       modified: Sequelize.DATE,
+      scanned: Sequelize.DATE,
       taken: Sequelize.DATE,
       width: Sequelize.INTEGER,
       height: Sequelize.INTEGER,
diff --git a/server/scanner.js b/server/scanner.js
index 7107d66..2729941 100755
--- a/server/scanner.js
+++ b/server/scanner.js
@@ -525,7 +525,9 @@ function scanDir(parent, path) {
         assets.push({
           path: path.slice(picturesPath.length),
           filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
-          stats: stats
+          name: file.replace(/.[^.]*$/, ""),
+          stats: stats,
+          parent: album
         });
       });
     });
@@ -538,6 +540,7 @@ function findOrCreateDBAlbum(album) {
   let query = "SELECT id FROM albums WHERE path=:path AND ";
   if (!album.parent) {
     query += "parentId IS NULL";
+    album.parentId = null;
   } else {
     if (!album.parent.id) {
       let error = "Albums in array in non ancestral order!";
@@ -553,8 +556,11 @@ function findOrCreateDBAlbum(album) {
     type: photoDB.sequelize.QueryTypes.SELECT
   }).then(function(results) {
     if (results.length == 0) {
-      return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parent,:name)", {
-        replacements: replacements
+      if (!album.parent) {
+        console.warn("Creating top level album: " + album.path);
+      }
+      return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
+        replacements: album
       }).then(function(results) {
         return results[1].lastID;
       });
@@ -562,7 +568,39 @@ function findOrCreateDBAlbum(album) {
       return results[0].id;
     }
   }).then(function(id) {
-    album.parentId = id;
+    album.id = id;
+    return id;
+  });
+}
+
+function findOrUpdateDBAsset(asset) {
+  let query = "SELECT id FROM photos WHERE albumId=:albumId AND filename=:filename";
+  if (!asset.parent || !asset.parent.id) {
+    let error = "Asset being processed without a parent";
+    console.error(error);
+    throw error;
+  }
+  asset.albumId = asset.parent.id;
+  return photoDB.sequelize.query(query, {
+    replacements: asset,
+    type: photoDB.sequelize.QueryTypes.SELECT
+  }).then(function(results) {
+    if (results.length == 0) {
+      /* Not in DB at all; HASH needs to be created. We could use the sharp loader to create
+       * a buffer, then pass that around. It would complicate our code, so instead we will
+       * leverage the OS' ability to cache read-only file assets in the cache and to pass
+       * those to the sharp layer later :) Accordingly, we will load the file here, create a SHA
+       * and create that HASH entry (unless it already exists, in which case DUPLICATE!!!! Gah.) */
+      return photoDB.sequelize.query("INSERT INTO photos (albumId,filename,name) VALUES(:albumId,:filename,:name)", {
+        replacements: asset
+      }).then(function(results) {
+        return results[1].lastID;
+      });
+    } else {
+      return results[0].id;
+    }
+  }).then(function(id) {
+    asset.id = id;
     return id;
   });
 }
@@ -572,6 +610,8 @@ module.exports = {
     photoDB = db;
     /* 1. Scan for all assets which will be managed by the system. readdir
      * 2. Check if entry in DB. Check mod-time in DB vs. stats from #1
+     *  - For albums
+     *  - For assets
      * 3. If not in DB, or mod-time changed, compute HASH of the file
      * 4. Check for HASH in photohash -- skip?
      * 5. Check for and create thumbs/FILE thumbs/scaled/FILE
@@ -584,10 +624,35 @@ module.exports = {
     return scanDir(null, picturesPath).spread(function(albums, assets) {
       console.log("Found " + assets.length + " assets in " + albums.length + " albums after " + 
         ((Date.now() - now) / 1000) + "s");
-      return Promise.map(albums, function(album) {
-        return db.sequelize
-      }, {
-        concurrency: 5
+      /* One at a time, in series, as the album[] array has parents first, then descendants.
+       * Operating in parallel could result in a child being searched for prior to the parent */
+      now = Date.now();
+
+      return Promise.mapSeries(albums, function(album) {
+        return findOrCreateDBAlbum(album);
+      }).then(function() {
+        console.log("Processed " + albums.length + " album DB entries in " + 
+          ((Date.now() - now) / 1000) + "s");
+        now = Date.now();
+
+        let processed = 0, start = Date.now(), last = 0;
+        return Promise.map(assets, function(asset) {
+          return findOrUpdateDBAsset(asset).then(function() {
+            let elapsed = Date.now() - start;
+            processed++;
+            if (elapsed > 5000) {
+              let remaining = assets.length - processed;
+              console.log(remaining + " assets remaining. ETA " + Math.ceil((elapsed / 1000) * remaining / (processed - last)) + "s");
+              last = processed;
+              start = Date.now();
+            }
+          });
+        }, {
+          concurrency: 5
+        }).then(function() {
+          console.log("Processed " + assets.length + " asset DB entries in " + 
+            ((Date.now() - now) / 1000) + "s");
+        });
       });
       /*triggerWatcher();*/
     });