Album and photo loading is all messed up

Signed-off-by: James Ketrenos <james_git@ketrenos.com>

frontend/elements/photo-thumbnail.html

@@ -101,7 +101,7 @@
         if (item === undefined|| base === undefined || item.path === undefined) {
             return "";
         }
-        return base + item.path + "/thumbs/" + item.filename;
+        return base + item.path + "thumbs/" + item.filename;
     },
 
     date: function(item) {
@@ -124,9 +124,9 @@
     attached: function() {
         var base = document.querySelector("base");
         if (base) {
-            this.base = new URL(base.href).pathname.replace(/\/$/, ""); /* Remove trailing slash if there */
+            this.base = new URL(base.href).pathname.replace(/\/$/, "") + "/"; /* Ensure trailing slash */
         } else {
-            this.base = "";
+            this.base = "/";
         }
     }
 });

frontend/src/ketr-photos/ketr-photos.html

@@ -32,7 +32,6 @@
 
 <link rel="import" href="../../elements/photo-lightbox.html">
 <link rel="import" href="../../elements/photo-thumbnail.html">
-<link rel="import" href="../../elements/pan-line.html">
 
 <script src="fetch.js"></script>
 
@@ -382,7 +381,8 @@
       </div>
     </paper-dialog>
     <paper-toast id="toast"></paper-toast>
-    <photo-lightbox tabindex="0" id="lightbox" on-close="lightBoxClose" on-next="lightBoxNext" on-previous="lightBoxPrevious"></photo-lightbox>
+    <photo-lightbox tabindex="0"
+      id="lightbox" on-close="lightBoxClose" on-next="lightBoxNext" on-previous="lightBoxPrevious"></photo-lightbox>
   </template>
 
   <script>
@@ -878,7 +878,7 @@
           }
 
           this.$.lightbox.item = el.item;
-          this.$.lightbox.src = this.base + el.item.path + "/thumbs/scaled/" + el.item.filename;
+          this.$.lightbox.src = this.base + el.item.path + "thumbs/scaled/" + el.item.filename;
           this.lightBoxElement = el;
           this.lightBoxElement.selected = true;
           this.disableScrolling = true;
@@ -943,7 +943,12 @@
 //            thumbnail.width = this.calcWidth;
             thumbnail.addEventListener("load-image", this._imageTap.bind(this));
             thumbnail.addEventListener("load-album", this.loadAlbum.bind(this));
-            datetime = new Date((photo.taken || photo.modified || photo.added) + " GMT").toISOString().replace(/T.*$/, "");
+            try {
+              datetime = new Date((photo.taken || photo.modified || photo.added).replace(/T.*/, "") + " GMT").toISOString().replace(/T.*$/, "");
+            } catch (error) {
+              console.log(JSON.stringify(photo, null, 2));
+              throw error;
+            }
 
             var dateBlock = this.root.querySelector("#date-" + datetime), thumbnails;
             if (!dateBlock) {
@@ -1043,7 +1048,7 @@
             query += key + "=" + encodeURIComponent(params[key]);
           }
 
-          var path = this.path || "", mode = this.mode;
+          var path = this.path || "/", mode = this.mode;
           if (mode != "albums") {
             path = "/" + mode;
             if (mode == "time") {
@@ -1053,7 +1058,7 @@
             }
           }
           var username = this.user ? this.user.username : "";
-          console.log("Requesting " + this.limit + " photos.");
+          console.log("Requesting " + this.limit + " photos from " + path);
           window.fetch("api/v1/photos" + path + query, function(path, error, xhr) {
             this.loading = false;
 
@@ -1063,7 +1068,7 @@
 
             if ((username != (this.user ? this.user.username : "")) ||
                 (mode != this.mode) || 
-                ((mode == "albums") && (path != (this.path || ""))) ||
+                ((mode == "albums") && (path != (this.path || "/"))) ||
                 ((mode == "memories") && (path != ("/memories/" + (this.date || ""))))) {
               console.log("Skipping results for old query. Triggering re-fetch of photos for new path or mode.");
               this._loadPhotos();
@@ -1089,9 +1094,9 @@
 
             var base = document.querySelector("base");
             if (base) {
-              this.base = new URL(base.href).pathname.replace(/\/$/, ""); /* Remove trailing slash if there */
+              this.base = new URL(base.href).pathname.replace(/\/$/, "") + "/"; /* Make sure there is a trailing slash */
             } else {
-              this.base = "";
+              this.base = "/";
             }
 
             console.log(results.items.length + " photos received.");
@@ -1112,7 +1117,7 @@
           }
           this.loadingAlbums = true;
 
-          var path = this.path || "";
+          var path = this.path || "/";
           window.fetch("api/v1/albums" + path, function(path, error, xhr) {
             this.loadingAlbums = false;
             
@@ -1120,14 +1125,14 @@
               return;
             }
             
-            if (path != (this.path || "")) {
+            if (path != (this.path || "/")) {
               console.log("Skipping results for old query. Triggering re-fetch of albums for new path.");
               this._loadAlbums();
               return;
             }
 
             if (error) {
-              console.log("Error loading album: " + (this.path || ""));
+              console.log("Error loading album: " + (this.path || "/"));
               console.error(JSON.stringify(error, null, 2));
               return;
             }

server/routes/albums.js

@@ -15,9 +15,9 @@ require("../db/photos").then(function(db) {
 const router = express.Router();
 
 router.get("/*", function(req, res/*, next*/) {
-  let url = decodeURI(req.url).replace(/\?.*$/, ""),
+  let url = decodeURI(req.url).replace(/\?.*$/, "").replace(/^\//, ""),
     query = "SELECT * FROM albums WHERE path=:path";
-  
+  console.log("Looking up album: " + url);
   return photoDB.sequelize.query(query, {
     replacements: {
       path: url

server/routes/photos.js

@@ -40,11 +40,11 @@ router.get("/memories/*", function(req, res/*, next*/) {
   if (id == -1) {
     index = "";
   } else {
-    index = " AND ((taken=DATE(:cursor) AND id<"+id+ ") OR taken<DATE(:cursor))";
+    index = " AND ((taken=DATE(:cursor) AND photos.id<"+id+ ") OR taken<DATE(:cursor))";
   }
 
   let date = new Date(decodeURI(req.url).replace(/\?.*$/, ""));
-  let query = "SELECT * FROM photos WHERE strftime('%m%d',taken)=strftime('%m%d',:date) " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
+  let query = "SELECT photos.*,albums.path AS path FROM photos INNER JOIN albums ON albums.id=photos.albumId WHERE strftime('%m%d',taken)=strftime('%m%d',:date) " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
 
 //  console.log("Memories for " + date.toISOString().replace(/T.*/, ""));
 //  console.log(query);
@@ -109,11 +109,11 @@ router.get("/*", function(req, res/*, next*/) {
   if (id == -1) {
     index = "";
   } else {
-    index = " AND ((taken=DATE(:cursor) AND id<"+id+ ") OR taken<DATE(:cursor))";
+    index = " AND ((taken=DATE(:cursor) AND photos.id<"+id+ ") OR taken<DATE(:cursor))";
   }
 
   let path = decodeURI(req.url).replace(/\?.*$/, ""),
-    query = "SELECT photos.* FROM photos INNER JOIN albums ON albums.id=photos.albumId AND albums.path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
+    query = "SELECT photos.*,albums.path AS path FROM photos INNER JOIN albums ON albums.id=photos.albumId AND albums.path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
 
   console.log("Fetching from: " + path);
   

server/scanner.js

@@ -195,8 +195,12 @@ function processBlock(items) {
     processQueue = processQueue.concat(items);
   }
   
-  if (!processRunning && processQueue.length) {
+  if (processRunning || processQueue.length == 0) {
-    let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0);
+    return;
+  }
+
+  let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0),
+    needsProcessing = [], duplicates = [];
   processRunning = true;
   /* Sort to newest files to be processed first */
   processing.sort(function(a, b) {
@@ -204,6 +208,39 @@ function processBlock(items) {
   });
 
   return Promise.map(processing, function(asset) {
+    return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
+      asset.hash = hash;
+    });
+  }).then(function() {
+    /* Needs to be one at a time in case there are multiple HASH collisions */
+    return Promise.mapSeries(processing, function(asset) {
+      return photoDB.sequelize.query("SELECT * FROM photohashes WHERE hash=:hash OR photoId=:id", {
+        replacements: asset,
+        type: photoDB.sequelize.QueryTypes.SELECT
+      }).then(function(results) {
+        let query;
+        if (results.length == 0) {
+          query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
+        } else if (results[0].hash != asset.hash) {
+          query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
+        } else if (results[0].photoId != asset.id) {
+          console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename);
+          duplicates.push(asset);
+          return;
+        } else {
+          return;
+        }
+
+        return photoDB.sequelize.query(query, {
+          replacements: asset
+        }).then(function() {
+          /* HASH has been updated; add to the needsProcessing array */
+          needsProcessing.push(asset);
+        });
+      });
+    });
+  }).then(function() {
+    return Promise.map(needsProcessing, function(asset) {
       var path = asset.album.path,
         file = asset.filename,
         created = asset.stats.ctime,
@@ -308,8 +345,7 @@ function processBlock(items) {
             });
           }).then(function() {
             return photoDB.sequelize.query("UPDATE photos SET " +
-              "(added,modified,taken,width,height,scanned)" +
+              "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP", {
-              "VALUES(:added,:modified,:taken,:width,:height,CURRENT_TIMESTAMP)", {
               replacements: asset
             });
           }).then(function() {
@@ -349,10 +385,10 @@ function processBlock(items) {
       });
     }, {
       concurrency: 1
-    }).then(function() {
-      console.log("Completed processing queue.");
     });
-  }
+  }).then(function() {
+    console.log("Completed processing queue. " + duplicates.length + " duplicates.");
+  });
 }
 
 
@@ -360,7 +396,7 @@ function scanDir(parent, path) {
   let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
     album = {
       path: path.slice(picturesPath.length), /* path already ends in '/' */
-      name: path.replace(/.*\//, "").replace(/_/g, " "),
+      name: path.replace(/\/$/, "").replace(/.*\//, "").replace(/_/g, " "),
       parent: parent,
       allAssetCount: 0,
       allAlbumCount: 0
@@ -458,7 +494,7 @@ function findOrCreateDBAlbum(album) {
   }).then(function(results) {
     if (results.length == 0) {
       if (!album.parent) {
-        console.warn("Creating top level album: " + album.path);
+        console.warn("Creating top level album: " + picturesPath);
       }
       return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
         replacements: album
@@ -540,17 +576,20 @@ module.exports = {
      * 2. Check if entry in DB. Check mod-time in DB vs. stats from #1
      *  - For albums
      *  - For assets
-     * 3. If not in DB, or mod-time changed, compute HASH of the file
+     * 3. If not in DB, or mod-time changed, queue for HASH CHECK
-     * 4. Check for HASH in photohash -- skip?
+     * 
-     * 5. Check for and create thumbs/FILE thumbs/scaled/FILE
+     * HASH CHECK
-     * 6. If necessary, create JPG from RAW
+     * 1. Compute HASH
-     * 7. Update last-scanned date in DB for entry
+     * 2. Check for HASH in photohash -- skip?
-     * 8. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were 
+     * 3. Check for and create thumbs/FILE thumbs/scaled/FILE
+     * 4. If necessary, create JPG from RAW
+     * 5. Update last-scanned date in DB for entry
+     * 6. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were 
      *    removed on disk)? Also purge from the HASH table.
      */
     let initialized = Date.now();
     let now = Date.now();
-    const needsProcessing = [], duplicates = [];
+    const needsProcessing = [];
     return scanDir(null, picturesPath).spread(function(albums, assets) {
       console.log("Found " + assets.length + " assets in " + albums.length + " albums after " + 
         ((Date.now() - now) / 1000) + "s");
@@ -565,11 +604,11 @@ module.exports = {
           ((Date.now() - now) / 1000) + "s");
         now = Date.now();
 
-        let processed = 0, start = Date.now(), last = 0, hashNeeded = [];
+        let processed = 0, start = Date.now(), last = 0;
         return Promise.map(assets, function(asset) {
           return findOrUpdateDBAsset(asset).then(function(asset) {
             if (asset.scanned < asset.stats.mtime) {
-              hashNeeded.push(asset);
+              needsProcessing.push(asset);
             }
           }).then(function(asset) {
             processed++;
@@ -588,46 +627,11 @@ module.exports = {
         }, {
           concurrency: 5
         }).then(function() {
-          console.log(hashNeeded.length + " assets need HASH computed");
+          console.log(needsProcessing.length + " assets need HASH computed");
-          return Promise.map(hashNeeded, function(asset) {
-            return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
-              asset.hash = hash;
-            });
-          }).then(function() {
-            /* Needs to be one at a time in case there are multiple HASH collisions */
-            return Promise.mapSeries(hashNeeded, function(asset) {
-              return db.sequelize.query("SELECT * FROM photohashes WHERE hash=:hash OR photoId=:id", {
-                replacements: asset,
-                type: photoDB.sequelize.QueryTypes.SELECT
-              }).then(function(results) {
-                let query;
-                if (results.length == 0) {
-                  query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
-                } else if (results[0].hash != asset.hash) {
-                  query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
-                } else if (results[0].photoId != asset.id) {
-                  console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename);
-                  duplicates.push(asset);
-                  return;
-                } else {
-                  return;
-                }
-
-                return db.sequelize.query(query, {
-                  replacements: asset
-                }).then(function() {
-                  /* HASH has been updated; add to the needsProcessing array */
-                  needsProcessing.push(asset);
-                });
-              });
-            }).then(function() {
           processBlock(needsProcessing);
-            });
-          });
         }).then(function() {
-          console.log("Processed " + assets.length + " asset DB entries in " + 
+          console.log("Scanned " + assets.length + " asset DB entries in " + 
             ((Date.now() - now) / 1000) + "s");
-          console.log(duplicates.length + " duplicates.");
         });
       });
     }).then(function() {