Scanning and everything now works

Signed-off-by: James Ketrenos <james_git@ketrenos.com>

frontend/src/ketr-photos/ketr-photos.html

@@ -655,11 +655,11 @@
         },
 
         breadcrumb: function(path) {
-          var crumbs = path.split("/"), parts = [];
+          var crumbs = path.replace(/(.*)\/$/, "/$1").split("/"), parts = [];
           path = "";
           crumbs.forEach(function(crumb, index) {
-            if (crumb) {
+            if (index > 0) {
-              path += "/" + crumb;
+              path += crumb + "/";
             }
             parts.push({
               name: crumb ? crumb : "Top",
@@ -736,7 +736,7 @@
           if (top) {
             var photo = top.item;
             this.$.pager.style.opacity = 1;
-            var date = window.moment(new Date((photo.taken || photo.modified || photo.added) + " GMT"));
+            var date = window.moment(new Date((photo.taken || photo.modified || photo.added).replace(/T.*/, " GMT")));
             date = date.calendar(null, { sameElse: "MMM DD, YYYY" }).replace(/ at.*/, "");
             this.$.pager.textContent = date;
             this.$.pager.style.top = headerHeight +
@@ -966,11 +966,9 @@
             return;
           }
 
-          var i = 0;
           photos.forEach(function(photo) {
-            var datetime = new Date((photo.taken || photo.modified || photo.added) + " GMT"),
+            var year = (photo.taken || photo.modified || photo.added).replace(/^(....).*$/, "$1");
-              year = datetime.getFullYear();
+            for (var i = 0; i < this.years.length; i++) {
-            for (; i < this.years.length; i++) {
               if (this.years[i] == year) {
                 return;
               }
@@ -1008,7 +1006,7 @@
             thumbnail.addEventListener("load-image", this._imageTap.bind(this));
             thumbnail.addEventListener("load-album", this.loadAlbum.bind(this));
             try {
-              datetime = new Date((photo.taken || photo.modified || photo.added).replace(/T.*/, "") + " GMT").toISOString().replace(/T.*$/, "");
+              datetime = (photo.taken || photo.modified || photo.added).replace(/T.*/, "");
             } catch (error) {
               console.log(JSON.stringify(photo, null, 2));
               throw error;
@@ -1112,18 +1110,18 @@
             query += key + "=" + encodeURIComponent(params[key]);
           }
 
-          var path = this.path || "/", mode = this.mode;
+          var path = this.path || "", mode = this.mode;
           if (mode != "albums") {
-            path = "/" + mode;
+            path = mode;
             if (mode == "time") {
               path = "";
             } else {
-              path = "/memories/" + (this.date || "");
+              path = "memories/" + (this.date || "");
             }
           }
           var username = this.user ? this.user.username : "";
           console.log("Requesting " + this.limit + " photos from " + path);
-          window.fetch("api/v1/photos" + path + query, function(path, error, xhr) {
+          window.fetch("api/v1/photos/" + path + query, function(path, error, xhr) {
             this.loading = false;
 
             if (!this.user) {
@@ -1132,8 +1130,8 @@
 
             if ((username != (this.user ? this.user.username : "")) ||
                 (mode != this.mode) || 
-                ((mode == "albums") && (path != (this.path || "/"))) ||
+                ((mode == "albums") && (path != (this.path || ""))) ||
-                ((mode == "memories") && (path != ("/memories/" + (this.date || ""))))) {
+                ((mode == "memories") && (path != ("memories/" + (this.date || ""))))) {
               console.log("Skipping results for old query. Triggering re-fetch of photos for new path or mode.");
               this._loadPhotos();
               return;
@@ -1181,22 +1179,22 @@
           }
           this.loadingAlbums = true;
 
-          var path = this.path || "/";
+          var path = this.path || "";
-          window.fetch("api/v1/albums" + path, function(path, error, xhr) {
+          window.fetch("api/v1/albums/" + path, function(path, error, xhr) {
             this.loadingAlbums = false;
             
             if (!this.user) {
               return;
             }
             
-            if (path != (this.path || "/")) {
+            if (path != (this.path || "")) {
               console.log("Skipping results for old query. Triggering re-fetch of albums for new path.");
               this._loadAlbums();
               return;
             }
 
             if (error) {
-              console.log("Error loading album: " + (this.path || "/"));
+              console.log("Error loading album: " + (this.path || ""));
               console.error(JSON.stringify(error, null, 2));
               return;
             }

server/app.js

@@ -41,8 +41,6 @@ app.set("trust proxy", true);
 /* Handle static files first so excessive logging doesn't occur */
 app.use(basePath, express.static("frontend", { index: false }));
 
-app.use(morgan("common"));
-
 app.use(bodyParser.json());
 app.use(bodyParser.urlencoded({
   extended: false
@@ -109,6 +107,9 @@ app.use(basePath, function(req, res, next) {
 });    
 
 app.use(basePath, express.static(picturesPath, { index: false }));
+
+app.use(morgan("common"));
+
 app.use(basePath + "api/v1/photos", require("./routes/photos"));
 app.use(basePath + "api/v1/days", require("./routes/days"));
 app.use(basePath + "api/v1/albums", require("./routes/albums"));

server/db/photos.js

@@ -60,6 +60,10 @@ function init() {
       taken: Sequelize.DATE,
       width: Sequelize.INTEGER,
       height: Sequelize.INTEGER,
+      duplicate: {
+        type: Sequelize.BOOLEAN,
+        defaultValue: 0
+      },
       albumId: {
         type: Sequelize.INTEGER,
         allowNull: true,

server/routes/albums.js

@@ -17,7 +17,7 @@ const router = express.Router();
 router.get("/*", function(req, res/*, next*/) {
   let url = decodeURI(req.url).replace(/\?.*$/, "").replace(/^\//, ""),
     query = "SELECT * FROM albums WHERE path=:path";
-  console.log("Looking up album: " + url);
+//  console.log("Looking up album: " + url);
   return photoDB.sequelize.query(query, {
     replacements: {
       path: url

server/routes/photos.js

@@ -38,13 +38,15 @@ router.get("/memories/*", function(req, res/*, next*/) {
   }
 
   if (id == -1) {
-    index = "";
+    index = "strftime('%m%d',taken)=strftime('%m%d',:date)";
   } else {
-    index = " AND ((taken=DATE(:cursor) AND photos.id<"+id+ ") OR taken<DATE(:cursor))";
+    index = "(strftime('%m%d',taken)=strftime('%m%d',:date) AND photos.id<"+id+ ")";
   }
 
   let date = new Date(decodeURI(req.url).replace(/\?.*$/, ""));
-  let query = "SELECT photos.*,albums.path AS path FROM photos INNER JOIN albums ON albums.id=photos.albumId WHERE strftime('%m%d',taken)=strftime('%m%d',:date) " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
+  let query = "SELECT photos.*,albums.path AS path FROM photos " +
+    "INNER JOIN albums ON (albums.id=photos.albumId) " +
+    "WHERE (photos.duplicate=0 AND photos.scanned NOT NULL AND " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
 
 //  console.log("Memories for " + date.toISOString().replace(/T.*/, ""));
 //  console.log(query);
@@ -109,19 +111,22 @@ router.get("/*", function(req, res/*, next*/) {
   if (id == -1) {
     index = "";
   } else {
-    index = " AND ((taken=DATE(:cursor) AND photos.id<"+id+ ") OR taken<DATE(:cursor))";
+    index = "AND ((strftime('%m%d',taken)=strftime('%m%d',:cursor) AND photos.id<"+id+ ") OR DATE(taken)<DATE(:cursor))";
   }
 
-  let path = decodeURI(req.url).replace(/\?.*$/, ""),
+  let path = decodeURI(req.url).replace(/\?.*$/, "").replace(/^\//, ""),
-    query = "SELECT photos.*,albums.path AS path FROM photos INNER JOIN albums ON albums.id=photos.albumId AND albums.path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
+    query = "SELECT photos.*,albums.path AS path FROM photos " +
-
+      "INNER JOIN albums ON (albums.id=photos.albumId AND albums.path LIKE :path) " +
-  console.log("Fetching from: " + path);
+      "WHERE (photos.duplicate=0 AND photos.scanned NOT NULL " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1),
-  
+      replacements = {
-  return photoDB.sequelize.query(query, {
-    replacements: {
         cursor: cursor,
         path: path + "%"
-    },
+      };
+
+//  console.log("Fetching from: " + JSON.stringify(replacements, null, 2) + "\nwith:\n" + query);
+  
+  return photoDB.sequelize.query(query, {
+    replacements: replacements,
     type: photoDB.Sequelize.QueryTypes.SELECT
   }).then(function(photos) {
     photos.forEach(function(photo) {

server/routes/users.js

@@ -101,6 +101,7 @@ router.post("/login", function(req, res) {
    * found there, we look them up in the site-specific user database */
    
   return ldapPromise(username, password).then(function(user) {
+    user.authenticated = 1;
     return user;
   }).catch(function() {
     console.log("User not found in LDAP. Looking up in DB.");

server/scanner.js

@@ -192,6 +192,7 @@ function processBlock(items) {
   setTimeout(processBlock, 1000);
 
   if (items) {
+    console.log("Adding " + items.length + " on top of " + processQueue.length + " assets.");
     processQueue = processQueue.concat(items);
   }
   
@@ -201,7 +202,9 @@ function processBlock(items) {
 
   let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0),
     needsProcessing = [], duplicates = [];
+
   processRunning = true;
+
   /* Sort to newest files to be processed first */
   processing.sort(function(a, b) {
     return a.stats.mtime - b.stats.mtime;
@@ -227,19 +230,22 @@ function processBlock(items) {
           console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename);
           duplicates.push(asset);
           return;
-        } else {
+        }
+
+        /* Even if the hash doesn't need to be updated, the entry needs to be scanned */
+        needsProcessing.push(asset);
+
+        if (!query) {
           return;
         }
 
         return photoDB.sequelize.query(query, {
           replacements: asset
-        }).then(function() {
-          /* HASH has been updated; add to the needsProcessing array */
-          needsProcessing.push(asset);
         });
       });
     });
   }).then(function() {
+    console.log(needsProcessing.length + " assets need to have metadata extracted");
     return Promise.map(needsProcessing, function(asset) {
       var path = asset.album.path,
         file = asset.filename,
@@ -343,7 +349,8 @@ function processBlock(items) {
             });
           }).then(function() {
             return photoDB.sequelize.query("UPDATE photos SET " +
-              "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP", {
+              "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP " +
+              "WHERE id=:id", {
               replacements: asset
             });
           }).then(function() {
@@ -385,7 +392,22 @@ function processBlock(items) {
       concurrency: 1
     });
   }).then(function() {
-    console.log("Completed processing queue. " + duplicates.length + " duplicates.");
+    console.log("Completed processing queue. Marking " + duplicates.length + " duplicates.");
+    let dups = [];
+    duplicates.forEach(function(asset) {
+      /* If not already marked as a duplicate, mark it. */
+      if (!asset.duplicate) {
+        dups.push(asset.id);
+      }
+    });
+
+    return photoDB.sequelize.query("UPDATE photos SET duplicate=1,scanned=CURRENT_TIME WHERE id IN (:dups)", {
+      replacements: {
+        dups: dups
+      }
+    });
+  }).then(function() {
+    processRunning = false;
   });
 }