From 6d234bdbc454322981c3f4120827d5493b9166df Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Sat, 18 Aug 2018 12:21:11 -0700 Subject: [PATCH] Updated to working Signed-off-by: James Ketrenos --- .gitignore | 2 +- config/default.json | 3 +- frontend/bower.json | 5 +- frontend/elements/photo-thumbnail.html | 101 +++++++++ frontend/src/ketr-photos/ketr-photos.html | 191 ++++++++++++----- package.json | 6 +- server/app.js | 6 + server/db/index.js | 39 +++- server/monkey.js | 29 +++ server/routes/photos.js | 91 +++++++- server/scanner.js | 246 ++++++++++++++++------ 11 files changed, 589 insertions(+), 130 deletions(-) create mode 100644 frontend/elements/photo-thumbnail.html create mode 100644 server/monkey.js diff --git a/.gitignore b/.gitignore index 0c8802e..240acb1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ node_modules -elements +./elements frontend/bower_components pictures diff --git a/config/default.json b/config/default.json index 92a8ad1..c7073a6 100644 --- a/config/default.json +++ b/config/default.json @@ -2,7 +2,8 @@ "db": { "host": "mysql://photos:p4$$w0rd@localhost:3306/photos", "options": { - "logging" : false + "logging" : false, + "timezone": "+00:00" } }, "server": { diff --git a/frontend/bower.json b/frontend/bower.json index 3ba5abf..4804cd3 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -36,7 +36,7 @@ "app-layout": "PolymerElements/app-layout#^0.9.1", "paper-checkbox": "PolymerElements/paper-checkbox#^1.2.0", "iron-form": "PolymerElements/iron-form#^1.0.16", - "iron-resizable-behavior": "PolymerElements/iron-resizable-behavior#^1.0.4", + "iron-resizable-behavior": "PolymerElements/iron-resizable-behavior#^2.1.1", "paper-dialog": "PolymerElements/paper-dialog#^1.1.0", "paper-dialog-scrollable": "PolymerElements/paper-dialog-scrollable#^1.1.5", "iron-collapse": "PolymerElements/iron-collapse#^1.2.1", @@ -53,6 +53,7 @@ "polymer": "^1.4.0", "iron-location": "^1.0.0", "iron-collapse": "^1.2.1", - "paper-spinner": "^1.0.0" + "paper-spinner": "^1.0.0", + "iron-resizable-behavior": "^1.0.4" } } diff --git a/frontend/elements/photo-thumbnail.html b/frontend/elements/photo-thumbnail.html new file mode 100644 index 0000000..a1f73d1 --- /dev/null +++ b/frontend/elements/photo-thumbnail.html @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + diff --git a/frontend/src/ketr-photos/ketr-photos.html b/frontend/src/ketr-photos/ketr-photos.html index fbf3e85..fe1838d 100755 --- a/frontend/src/ketr-photos/ketr-photos.html +++ b/frontend/src/ketr-photos/ketr-photos.html @@ -18,9 +18,12 @@ + + + + +
[[path]]
 -
- -
-
- +
+
+ +
+
+
+ prev + next +
+
+ +
- @@ -108,35 +112,88 @@ is: "ketr-photos", properties: { "loading": Boolean, - "photos": Array + "photos": Array, + prev: { + type: Boolean, + value: false + }, + next: { + type: Boolean, + value: false + } }, observers: [ ], + behaviors: [ + /* @polymerBehavior Polymer.IronResizableBehavior */ + Polymer.IronResizableBehavior + ], + + listeners: { + "iron-resize" : "onResize" + }, + + date: function(item) { + var datetime = item.taken || item.modified || item.added; + return datetime.replace(/T.*$/, ""); + }, + _imageTap: function(event) { - window.open(event.model.item.filepath, "image"); + window.open(this.base + event.model.item.path + "/" + event.model.item.filename, "image"); }, _pathTap: function(event) { window.location.href = event.model.item.filepath; }, - _loadPhotos: function() { + loadNextPhotos: function() { + var cursor = this.photos[this.photos.length - 1]; + this._loadPhotos(cursor.id + "_" + cursor.taken.toString().replace(/T.*/, ""), +1, true); + }, + + loadPrevPhotos: function() { + var cursor = this.photos[0]; + this._loadPhotos(cursor.id + "_" + cursor.taken.toString().replace(/T.*/, ""), -1); + }, + + _loadPhotos: function(start, dir, append) { if (this.loading == true) { return; } this.loading = true; - window.fetch("api/v1/photos", function(error, xhr) { + + dir = dir || +1; + var params = { + limit: Math.ceil(this.clientWidth / 200) * Math.ceil(this.clientHeight / 200), + dir: dir + }, url = ""; + if (start) { + params.next = start; + } + if (this.sortOrder) { + params.sort = this.sortOrder; + } + for (var key in params) { + if (url == "") { + url = "?"; + } else { + url += "&"; + } + url += key + "=" + encodeURIComponent(params[key]); + } + + window.fetch("api/v1/photos" + url, function(error, xhr) { this.loading = false; if (error) { console.error(JSON.stringify(error, null, 2)); return; } - let photos; + var results; try { - photos = JSON.parse(xhr.responseText); + results = JSON.parse(xhr.responseText); } catch (___) { this.$.toast.text = "Unable to load/parse photo list."; this.$.toast.setAttribute("error", true); @@ -148,42 +205,41 @@ var base = document.querySelector("base"); if (base) { - this.base = new URL(base.href).pathname; + this.base = new URL(base.href).pathname.replace(/\/$/, ""); /* Remove trailing slash if there */ } else { this.base = ""; } - photos.forEach(function(photo) { - photo.path = this.base + photo.path; - }.bind(this)); - - function findPath(path, item) { - if (path.indexOf(item.path) != 0) { - return false; - } - - if (path == item.path || path == item.path + "/") { - return item; - } - - for (var i = 0; i < item.paths.length; i++) { - var tmp = findPath(path, item.paths[i]); - if (tmp) { - return tmp; - } - } - return false; + if (append) { + results.items.forEach(function(photo) { + this.push("photos", photo); + }.bind(this)); + } else { + this.photos = results.items; } - if (this.base != "") { - this.photos = findPath(this.base, photos) || photos; + if (dir == +1) { + this.prev = start ? true : false; + this.next = results.more ? true : false; } else { - this.photos = photos; + this.prev = results.more ? true : false; + this.next = true; } }.bind(this)); }, + onResize: function(event) { + this.debounce("resize", function() { + var width = Math.max(this.$.placeholder.offsetWidth || 0, 200), + cols = Math.floor(this.clientWidth / width), + calc = width + Math.floor((this.clientWidth % width) / cols); + if (calc != this.calcWidth) { + this.calcWidth = calc; + } + }, 100); + }, + ready: function() { window.addEventListener("hashchange", function(event) { this.hash = event.newURL.replace(/^[^#]*/, ""); @@ -197,7 +253,30 @@ } }.bind(this), 100); + window.setInterval(function() { + function isElementInViewport(el) { + var rect = el.getBoundingClientRect(), + vWidth = window.innerWidth || doc.documentElement.clientWidth, + vHeight = window.innerHeight || doc.documentElement.clientHeight; + + // Return false if it's not in the viewport + if (rect.right < 0 || rect.bottom < 0 + || rect.left > vWidth || rect.top > vHeight) { + return false; + } + + return true; + } + + if (this.next && isElementInViewport(this.$.magic)) { + this.loadNextPhotos(); + } + + }.bind(this), 500); + this._loadPhotos(); + + this.onResize(); } }); }); diff --git a/package.json b/package.json index d2e23f2..2419cca 100644 --- a/package.json +++ b/package.json @@ -19,13 +19,17 @@ "body-parser": "^1.18.2", "config": "^1.28.1", "cookie-parser": "^1.4.3", + "exif-reader": "github:paras20xx/exif-reader", "express": "^4.16.2", "mariasql": "^0.2.6", + "moment": "^2.22.2", "morgan": "^1.9.0", "mysql2": "^1.5.1", "node-inspector": "^1.1.1", + "qs": "^6.5.2", "sequelize": "^4.28.6", - "sequelize-mysql": "^1.7.0" + "sequelize-mysql": "^1.7.0", + "sharp": "^0.20.5" }, "jshintConfig": { "undef": true, diff --git a/server/app.js b/server/app.js index cf104d4..57080f8 100644 --- a/server/app.js +++ b/server/app.js @@ -1,5 +1,11 @@ "use strict"; +process.env.TZ = "Etc/GMT"; + +if (process.env.LOG_LINE) { + require("./monkey.js"); /* monkey patch console.log */ +} + console.log("Loading photos.ketr"); const express = require("express"), diff --git a/server/db/index.js b/server/db/index.js index 33f280e..4ab6924 100644 --- a/server/db/index.js +++ b/server/db/index.js @@ -27,12 +27,49 @@ function init() { console.log("DB initialization beginning. DB access will block."); return db.sequelize.authenticate().then(function () { + const Album = db.sequelize.define('album', { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true + }, + path: Sequelize.STRING, + parentId: { + type: Sequelize.INTEGER, + allowNull: true + } + }, { + classMethods: { + associate: function() { + Album.hasOne(Album, {as:'Album', foreignKey: 'parentId'}); + } + } + }); + const Photo = db.sequelize.define('photo', { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true + }, path: Sequelize.STRING, filename: Sequelize.STRING, - added: Sequelize.DATE + added: Sequelize.DATE, + modified: Sequelize.DATE, + taken: Sequelize.DATE, + width: Sequelize.INTEGER, + height: Sequelize.INTEGER, + albumId: { + type: Sequelize.INTEGER, + allowNull: true, + references: { + model: Album, + key: 'id', + } + } }); + console.log("Connection established successfully with DB."); return db.sequelize.sync({ force: false diff --git a/server/monkey.js b/server/monkey.js new file mode 100644 index 0000000..1c8d322 --- /dev/null +++ b/server/monkey.js @@ -0,0 +1,29 @@ +/* monkey-patch console.log to prefix with file/line-number */ +function lineLogger(logFn) { + let cwd = process.cwd(), + cwdRe = new RegExp("^[^/]*" + cwd.replace("/", "\\/") + "\/([^:]*:[0-9]*).*$"); + + function getErrorObject() { + try { + throw Error(); + } catch (err) { + return err; + } + } + + let err = getErrorObject(), + caller_line = err.stack.split("\n")[4], + args = [caller_line.replace(cwdRe, "$1 -")]; + + /* arguments.unshift() doesn't exist... */ + for (var i = 1; i < arguments.length; i++) { + args.push(arguments[i]); + } + + logFn.apply(this, args); +} + +console.log = lineLogger.bind(console, console.log); +console.warn = lineLogger.bind(console, console.warn); +console.error = lineLogger.bind(console, console.error); + diff --git a/server/routes/photos.js b/server/routes/photos.js index 7e87a35..08041d6 100644 --- a/server/routes/photos.js +++ b/server/routes/photos.js @@ -3,7 +3,8 @@ const express = require("express"), fs = require("fs"), url = require("url"), - config = require("config"); + config = require("config"), + moment = require("moment"); let photoDB; @@ -24,13 +25,95 @@ const router = express.Router(); */ router.get("/", function(req, res/*, next*/) { - return photoDB.sequelize.query("SELECT path,filename,added FROM photos WHERE path LIKE :path", { + let limit = parseInt(req.query.limit) || 50, + order = (parseInt(req.query.dir) == -1) ? "DESC" : "", id, cursor, index; + + if (req.query.next) { + let parts = req.query.next.split("_"); + cursor = parts[1]; + id = parseInt(parts[0]); + } else { + cursor = ""; + id = -1; + } + + if (id == -1) { + index = ""; + } else { + if (order == "DESC") { + if (id != -1) { + index = " AND ((taken=DATE(:cursor) AND id<"+id+ ") OR takenDATE(:cursor))"; + } else { + index = " AND (taken>=DATE(:cursor))"; + } + } + } + + let query = "SELECT * FROM photos WHERE path LIKE :path " + index + " ORDER BY taken " + order + ",id " + order + " LIMIT " + (limit * 2 + 1); + return photoDB.sequelize.query(query, { replacements: { - path: req.url + "%" + cursor: cursor, + path: req.url.replace(/\?.*$/, "") + "%" }, type: photoDB.Sequelize.QueryTypes.SELECT }).then(function(photos) { - return res.status(200).json(photos); + photos.forEach(function(photo) { + for (var key in photo) { + if (photo[key] instanceof Date) { + photo[key].setHours(0, 0, 0, 0); + photo[key] = moment(photo[key]); + } + } + }); + + if (order == "DESC") { + if (cursor) { + photos = photos.filter(function(photo) { + if (!cursor.isSame(photo.taken, "day")) { + return true; + } + return photo.id < id; + }); + } + photos.reverse(); + } else { + if (cursor) { + cursor = moment(cursor); + photos = photos.filter(function(photo) { + if (!cursor.isSame(photo.taken, "day")) { + return true; + } + return photo.id > id; + }); + } + } + + let more = photos.length > limit; /* We queried one extra item to see if there are more than LIMIT available */ + + if (more) { + photos.slice(limit, photos.length); + } + photos.forEach(function(photo) { + photo.path = encodeURI(photo.path); + photo.filename = encodeURI(photo.filename); + }); + + let results = { + items: photos + }; + if (more) { + results.more = true; + } + return res.status(200).json(results); + }).catch(function(error) { + console.error("Query failed: " + query); + return Promise.reject(error); }); }); diff --git a/server/scanner.js b/server/scanner.js index 60f4a5d..d1abc82 100644 --- a/server/scanner.js +++ b/server/scanner.js @@ -2,7 +2,8 @@ const Promise = require("bluebird"), fs = require("fs"), - config = require("config"); + config = require("config"), + moment = require("moment"); let scanning = 0; @@ -10,79 +11,119 @@ let photoDB = null; const picturesPath = config.get("picturesPath"); -function scanFile(path, file, stats) { - return new Promise(function(resolve, reject) { - console.log("Scanning file: " + path + "/" + file); - return resolve(true); - }); -} +const processQueue = []; -function scanDir(path) { - return new Promise(function(resolve, reject) { - console.log("Scanning path " + path); +function scanDir(parent, path) { + let extensions = [ "jpg", "jpeg", "png", "gif" ], + re = "\.((" + extensions.join(")|(") + "))$"; + re = new RegExp(re, "i"); - fs.readdir(path, function(err, files) { - if (err) { - console.warn(" Could not readdir " + path); - return resolve(null); - } + return photoDB.sequelize.query("SELECT id FROM albums WHERE path=:path AND parentId=:parent", { + replacements: { + path: path, + parent: parent || null + }, + type: photoDB.sequelize.QueryTypes.SELECT + }).then(function(results) { + if (results.length == 0) { + console.log("Adding " + path + " under " + parent); + return photoDB.sequelize.query("INSERT INTO albums SET path=:path,parentId=:parent", { + replacements: { + path: path, + parent: parent || null + }, + }).then(function(results) { + return results[0]; + }); + } else { + return results[0].id; + } + }).then(function(parent) { + return new Promise(function(resolve, reject) { + // console.log("Scanning path " + path); - scanning++; + fs.readdir(path, function(err, files) { + if (err) { + console.warn(" Could not readdir " + path); + return resolve(null); + } - return Promise.map(files, function(file) { - let filepath = path + "/" + file; + scanning++; - return new Promise(function(resolve, reject) { - fs.stat(filepath, function(err, stats) { - if (err) { - console.warn("Could not stat " + filepath); - return resolve(false); - } + let hasThumbs = false; + for (let i = 0; i < files.length; i++) { + if (files[i] == "thumbs") { + hasThumbs = true; + break; + } + } - if (stats.isDirectory()) { - return scanDir(filepath, stats).then(function(entry) { - return resolve(true); - }); - } + let tmp = Promise.resolve(); - /* stats.isFile() */ - return scanFile(path, file, stats).then(function(entry) { - if (!entry) { - return resolve(false); + if (!hasThumbs) { + tmp = new Promise(function(resolve, reject) { + fs.mkdir(path + "/thumbs", function(err) { + if (err) { + return reject("Unable to create " + paths + "/thumbs"); } - - const replacements = { - path: path.slice(picturesPath.length), - filename: file, - added: new Date() - }; - - return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", { - replacements: replacements, - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(photo) { - if (photo.length == 0) { - return photoDB.sequelize.query("INSERT INTO photos " + - "SET path=:path,filename=:filename,added=DATE(:added)", { - replacements: replacements - }); - } - }).then(function() { - return resolve(true); - }); + return resolve(); }); }); - }); - }, { - concurrency: 10 - }).then(function() { - scanning--; - if (scanning == 0) { - const endStamp = Date.now(); - console.log("Scanning completed in " + Math.round(((endStamp - startStamp))) + "ms."); } - }).then(function() { - return resolve(); + + return tmp.then(function() { + return Promise.map(files, function(file) { + let filepath = path + "/" + file; + if (file == "thumbs") { + return resolve(true); + } + + return new Promise(function(resolve, reject) { + fs.stat(filepath, function(err, stats) { + if (err) { + console.warn("Could not stat " + filepath); + return resolve(false); + } + + if (stats.isDirectory()) { + return scanDir(parent, filepath, stats).then(function(entry) { + return resolve(true); + }); + } + + /* stats.isFile() */ + if (!re.exec(file)) { + return resolve(true); + } + + const replacements = { + path: path.slice(picturesPath.length), + filename: file + }; + + return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", { + replacements: replacements, + type: photoDB.sequelize.QueryTypes.SELECT + }).then(function(photo) { + if (photo.length == 0) { + processQueue.push([ replacements.path, file, stats.mtime, parent ]); + } + return resolve(true); + }); + }); + }); + }, { + concurrency: 10 + }).then(function() { + scanning--; + if (scanning == 0) { + const endStamp = Date.now(); + console.log("Scanning completed in " + Math.round(((endStamp - startStamp))) + "ms. " + processQueue.length + " items to process."); + } + }).then(function() { + return resolve(); + }); + }); }); }); }); @@ -90,9 +131,86 @@ function scanDir(path) { const startStamp = Date.now(); +let processRunning = false; + +const { spawn } = require('child_process'); + +const sharp = require("sharp"), exif = require("exif-reader"); + +function triggerWatcher() { + setTimeout(triggerWatcher, 1000); + + if (!processRunning && processQueue.length) { + processRunning = true; + return Promise.map(processQueue, function(entry) { + var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3], + src = picturesPath + path + "/" + file, + dst = picturesPath + path + "/thumbs/" + file, + image = sharp(src); +// console.log("Processing " + src); + return image.metadata().then(function(metadata) { + if (metadata.exif) { + metadata.exif = exif(metadata.exif); + delete metadata.exif.thumbnail; + delete metadata.exif.image; + for (var key in metadata.exif.exif) { + if (Buffer.isBuffer(metadata.exif.exif[key])) { + metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]"; + } + } + } + + let replacements = { + albumId: albumId, + path: path, + filename: file, + width: metadata.width, + height: metadata.height, + added: moment().format().replace(/T.*/, "") + }; + + if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) { + metadata.exif.exif.DateTimeOriginal.setHours(0, 0, 0, 0); + metadata.exif.exif.DateTimeOriginal = metadata.exif.exif.DateTimeOriginal.toISOString().replace(/T.*/, ""); +// console.log(metadata.exif.exif.DateTimeOriginal); + replacements.taken = moment(metadata.exif.exif.DateTimeOriginal, "YYYY-MM-DD").format().replace(/T.*/, ""); + replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format().replace(/T.*/, ""); + } else { +// console.log("Missing EXIF info for: " + file); + //console.log(JSON.stringify(metadata.exif, null, 2)); + let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = replacements.added; + let match = file.match(patterns); + if (match) { + date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format(); +// console.log("Constructed date: " + date); + } else { + date = moment(created).format(); +// console.log("Date from file: ", src, date); + } + replacements.taken = replacements.modified = date; + } + + return image.resize(256, 256).toFile(dst).then(function() { + return photoDB.sequelize.query("INSERT INTO photos " + + "SET albumId=:albumId,path=:path,filename=:filename,added=DATE(:added),modified=DATE(:modified),taken=DATE(:taken),width=:width,height=:height", { + replacements: replacements + }); + }).catch(function(error) { + console.log("Error resizing or writing " + src, error); + return Promise.Reject(); + }); + }); + }, { + concurrency: 1 + }); + } +} + module.exports = { scan: function (db) { photoDB = db; - return scanDir(picturesPath); + return scanDir(0, picturesPath).then(function() { + triggerWatcher(); + }); } };