-
- [[item.filepath]]
-
+
+
+
+
+
+
+ [[item.filepath]]
+
+
-
@@ -108,35 +112,88 @@
is: "ketr-photos",
properties: {
"loading": Boolean,
- "photos": Array
+ "photos": Array,
+ prev: {
+ type: Boolean,
+ value: false
+ },
+ next: {
+ type: Boolean,
+ value: false
+ }
},
observers: [
],
+ behaviors: [
+ /* @polymerBehavior Polymer.IronResizableBehavior */
+ Polymer.IronResizableBehavior
+ ],
+
+ listeners: {
+ "iron-resize" : "onResize"
+ },
+
+ date: function(item) {
+ var datetime = item.taken || item.modified || item.added;
+ return datetime.replace(/T.*$/, "");
+ },
+
_imageTap: function(event) {
- window.open(event.model.item.filepath, "image");
+ window.open(this.base + event.model.item.path + "/" + event.model.item.filename, "image");
},
_pathTap: function(event) {
window.location.href = event.model.item.filepath;
},
- _loadPhotos: function() {
+ loadNextPhotos: function() {
+ var cursor = this.photos[this.photos.length - 1];
+ this._loadPhotos(cursor.id + "_" + cursor.taken.toString().replace(/T.*/, ""), +1, true);
+ },
+
+ loadPrevPhotos: function() {
+ var cursor = this.photos[0];
+ this._loadPhotos(cursor.id + "_" + cursor.taken.toString().replace(/T.*/, ""), -1);
+ },
+
+ _loadPhotos: function(start, dir, append) {
if (this.loading == true) {
return;
}
this.loading = true;
- window.fetch("api/v1/photos", function(error, xhr) {
+
+ dir = dir || +1;
+ var params = {
+ limit: Math.ceil(this.clientWidth / 200) * Math.ceil(this.clientHeight / 200),
+ dir: dir
+ }, url = "";
+ if (start) {
+ params.next = start;
+ }
+ if (this.sortOrder) {
+ params.sort = this.sortOrder;
+ }
+ for (var key in params) {
+ if (url == "") {
+ url = "?";
+ } else {
+ url += "&";
+ }
+ url += key + "=" + encodeURIComponent(params[key]);
+ }
+
+ window.fetch("api/v1/photos" + url, function(error, xhr) {
this.loading = false;
if (error) {
console.error(JSON.stringify(error, null, 2));
return;
}
- let photos;
+ var results;
try {
- photos = JSON.parse(xhr.responseText);
+ results = JSON.parse(xhr.responseText);
} catch (___) {
this.$.toast.text = "Unable to load/parse photo list.";
this.$.toast.setAttribute("error", true);
@@ -148,42 +205,41 @@
var base = document.querySelector("base");
if (base) {
- this.base = new URL(base.href).pathname;
+ this.base = new URL(base.href).pathname.replace(/\/$/, ""); /* Remove trailing slash if there */
} else {
this.base = "";
}
- photos.forEach(function(photo) {
- photo.path = this.base + photo.path;
- }.bind(this));
-
- function findPath(path, item) {
- if (path.indexOf(item.path) != 0) {
- return false;
- }
-
- if (path == item.path || path == item.path + "/") {
- return item;
- }
-
- for (var i = 0; i < item.paths.length; i++) {
- var tmp = findPath(path, item.paths[i]);
- if (tmp) {
- return tmp;
- }
- }
- return false;
+ if (append) {
+ results.items.forEach(function(photo) {
+ this.push("photos", photo);
+ }.bind(this));
+ } else {
+ this.photos = results.items;
}
- if (this.base != "") {
- this.photos = findPath(this.base, photos) || photos;
+ if (dir == +1) {
+ this.prev = start ? true : false;
+ this.next = results.more ? true : false;
} else {
- this.photos = photos;
+ this.prev = results.more ? true : false;
+ this.next = true;
}
}.bind(this));
},
+ onResize: function(event) {
+ this.debounce("resize", function() {
+ var width = Math.max(this.$.placeholder.offsetWidth || 0, 200),
+ cols = Math.floor(this.clientWidth / width),
+ calc = width + Math.floor((this.clientWidth % width) / cols);
+ if (calc != this.calcWidth) {
+ this.calcWidth = calc;
+ }
+ }, 100);
+ },
+
ready: function() {
window.addEventListener("hashchange", function(event) {
this.hash = event.newURL.replace(/^[^#]*/, "");
@@ -197,7 +253,30 @@
}
}.bind(this), 100);
+ window.setInterval(function() {
+ function isElementInViewport(el) {
+ var rect = el.getBoundingClientRect(),
+ vWidth = window.innerWidth || doc.documentElement.clientWidth,
+ vHeight = window.innerHeight || doc.documentElement.clientHeight;
+
+ // Return false if it's not in the viewport
+ if (rect.right < 0 || rect.bottom < 0
+ || rect.left > vWidth || rect.top > vHeight) {
+ return false;
+ }
+
+ return true;
+ }
+
+ if (this.next && isElementInViewport(this.$.magic)) {
+ this.loadNextPhotos();
+ }
+
+ }.bind(this), 500);
+
this._loadPhotos();
+
+ this.onResize();
}
});
});
diff --git a/package.json b/package.json
index d2e23f2..2419cca 100644
--- a/package.json
+++ b/package.json
@@ -19,13 +19,17 @@
"body-parser": "^1.18.2",
"config": "^1.28.1",
"cookie-parser": "^1.4.3",
+ "exif-reader": "github:paras20xx/exif-reader",
"express": "^4.16.2",
"mariasql": "^0.2.6",
+ "moment": "^2.22.2",
"morgan": "^1.9.0",
"mysql2": "^1.5.1",
"node-inspector": "^1.1.1",
+ "qs": "^6.5.2",
"sequelize": "^4.28.6",
- "sequelize-mysql": "^1.7.0"
+ "sequelize-mysql": "^1.7.0",
+ "sharp": "^0.20.5"
},
"jshintConfig": {
"undef": true,
diff --git a/server/app.js b/server/app.js
index cf104d4..57080f8 100644
--- a/server/app.js
+++ b/server/app.js
@@ -1,5 +1,11 @@
"use strict";
+process.env.TZ = "Etc/GMT";
+
+if (process.env.LOG_LINE) {
+ require("./monkey.js"); /* monkey patch console.log */
+}
+
console.log("Loading photos.ketr");
const express = require("express"),
diff --git a/server/db/index.js b/server/db/index.js
index 33f280e..4ab6924 100644
--- a/server/db/index.js
+++ b/server/db/index.js
@@ -27,12 +27,49 @@ function init() {
console.log("DB initialization beginning. DB access will block.");
return db.sequelize.authenticate().then(function () {
+ const Album = db.sequelize.define('album', {
+ id: {
+ type: Sequelize.INTEGER,
+ primaryKey: true,
+ autoIncrement: true
+ },
+ path: Sequelize.STRING,
+ parentId: {
+ type: Sequelize.INTEGER,
+ allowNull: true
+ }
+ }, {
+ classMethods: {
+ associate: function() {
+ Album.hasOne(Album, {as:'Album', foreignKey: 'parentId'});
+ }
+ }
+ });
+
const Photo = db.sequelize.define('photo', {
+ id: {
+ type: Sequelize.INTEGER,
+ primaryKey: true,
+ autoIncrement: true
+ },
path: Sequelize.STRING,
filename: Sequelize.STRING,
- added: Sequelize.DATE
+ added: Sequelize.DATE,
+ modified: Sequelize.DATE,
+ taken: Sequelize.DATE,
+ width: Sequelize.INTEGER,
+ height: Sequelize.INTEGER,
+ albumId: {
+ type: Sequelize.INTEGER,
+ allowNull: true,
+ references: {
+ model: Album,
+ key: 'id',
+ }
+ }
});
+
console.log("Connection established successfully with DB.");
return db.sequelize.sync({
force: false
diff --git a/server/monkey.js b/server/monkey.js
new file mode 100644
index 0000000..1c8d322
--- /dev/null
+++ b/server/monkey.js
@@ -0,0 +1,29 @@
+/* monkey-patch console.log to prefix with file/line-number */
+function lineLogger(logFn) {
+ let cwd = process.cwd(),
+ cwdRe = new RegExp("^[^/]*" + cwd.replace("/", "\\/") + "\/([^:]*:[0-9]*).*$");
+
+ function getErrorObject() {
+ try {
+ throw Error();
+ } catch (err) {
+ return err;
+ }
+ }
+
+ let err = getErrorObject(),
+ caller_line = err.stack.split("\n")[4],
+ args = [caller_line.replace(cwdRe, "$1 -")];
+
+ /* arguments.unshift() doesn't exist... */
+ for (var i = 1; i < arguments.length; i++) {
+ args.push(arguments[i]);
+ }
+
+ logFn.apply(this, args);
+}
+
+console.log = lineLogger.bind(console, console.log);
+console.warn = lineLogger.bind(console, console.warn);
+console.error = lineLogger.bind(console, console.error);
+
diff --git a/server/routes/photos.js b/server/routes/photos.js
index 7e87a35..08041d6 100644
--- a/server/routes/photos.js
+++ b/server/routes/photos.js
@@ -3,7 +3,8 @@
const express = require("express"),
fs = require("fs"),
url = require("url"),
- config = require("config");
+ config = require("config"),
+ moment = require("moment");
let photoDB;
@@ -24,13 +25,95 @@ const router = express.Router();
*/
router.get("/", function(req, res/*, next*/) {
- return photoDB.sequelize.query("SELECT path,filename,added FROM photos WHERE path LIKE :path", {
+ let limit = parseInt(req.query.limit) || 50,
+ order = (parseInt(req.query.dir) == -1) ? "DESC" : "", id, cursor, index;
+
+ if (req.query.next) {
+ let parts = req.query.next.split("_");
+ cursor = parts[1];
+ id = parseInt(parts[0]);
+ } else {
+ cursor = "";
+ id = -1;
+ }
+
+ if (id == -1) {
+ index = "";
+ } else {
+ if (order == "DESC") {
+ if (id != -1) {
+ index = " AND ((taken=DATE(:cursor) AND id<"+id+ ") OR taken
DATE(:cursor))";
+ } else {
+ index = " AND (taken>=DATE(:cursor))";
+ }
+ }
+ }
+
+ let query = "SELECT * FROM photos WHERE path LIKE :path " + index + " ORDER BY taken " + order + ",id " + order + " LIMIT " + (limit * 2 + 1);
+ return photoDB.sequelize.query(query, {
replacements: {
- path: req.url + "%"
+ cursor: cursor,
+ path: req.url.replace(/\?.*$/, "") + "%"
},
type: photoDB.Sequelize.QueryTypes.SELECT
}).then(function(photos) {
- return res.status(200).json(photos);
+ photos.forEach(function(photo) {
+ for (var key in photo) {
+ if (photo[key] instanceof Date) {
+ photo[key].setHours(0, 0, 0, 0);
+ photo[key] = moment(photo[key]);
+ }
+ }
+ });
+
+ if (order == "DESC") {
+ if (cursor) {
+ photos = photos.filter(function(photo) {
+ if (!cursor.isSame(photo.taken, "day")) {
+ return true;
+ }
+ return photo.id < id;
+ });
+ }
+ photos.reverse();
+ } else {
+ if (cursor) {
+ cursor = moment(cursor);
+ photos = photos.filter(function(photo) {
+ if (!cursor.isSame(photo.taken, "day")) {
+ return true;
+ }
+ return photo.id > id;
+ });
+ }
+ }
+
+ let more = photos.length > limit; /* We queried one extra item to see if there are more than LIMIT available */
+
+ if (more) {
+ photos.slice(limit, photos.length);
+ }
+ photos.forEach(function(photo) {
+ photo.path = encodeURI(photo.path);
+ photo.filename = encodeURI(photo.filename);
+ });
+
+ let results = {
+ items: photos
+ };
+ if (more) {
+ results.more = true;
+ }
+ return res.status(200).json(results);
+ }).catch(function(error) {
+ console.error("Query failed: " + query);
+ return Promise.reject(error);
});
});
diff --git a/server/scanner.js b/server/scanner.js
index 60f4a5d..d1abc82 100644
--- a/server/scanner.js
+++ b/server/scanner.js
@@ -2,7 +2,8 @@
const Promise = require("bluebird"),
fs = require("fs"),
- config = require("config");
+ config = require("config"),
+ moment = require("moment");
let scanning = 0;
@@ -10,79 +11,119 @@ let photoDB = null;
const picturesPath = config.get("picturesPath");
-function scanFile(path, file, stats) {
- return new Promise(function(resolve, reject) {
- console.log("Scanning file: " + path + "/" + file);
- return resolve(true);
- });
-}
+const processQueue = [];
-function scanDir(path) {
- return new Promise(function(resolve, reject) {
- console.log("Scanning path " + path);
+function scanDir(parent, path) {
+ let extensions = [ "jpg", "jpeg", "png", "gif" ],
+ re = "\.((" + extensions.join(")|(") + "))$";
+ re = new RegExp(re, "i");
- fs.readdir(path, function(err, files) {
- if (err) {
- console.warn(" Could not readdir " + path);
- return resolve(null);
- }
+ return photoDB.sequelize.query("SELECT id FROM albums WHERE path=:path AND parentId=:parent", {
+ replacements: {
+ path: path,
+ parent: parent || null
+ },
+ type: photoDB.sequelize.QueryTypes.SELECT
+ }).then(function(results) {
+ if (results.length == 0) {
+ console.log("Adding " + path + " under " + parent);
+ return photoDB.sequelize.query("INSERT INTO albums SET path=:path,parentId=:parent", {
+ replacements: {
+ path: path,
+ parent: parent || null
+ },
+ }).then(function(results) {
+ return results[0];
+ });
+ } else {
+ return results[0].id;
+ }
+ }).then(function(parent) {
+ return new Promise(function(resolve, reject) {
+ // console.log("Scanning path " + path);
- scanning++;
+ fs.readdir(path, function(err, files) {
+ if (err) {
+ console.warn(" Could not readdir " + path);
+ return resolve(null);
+ }
- return Promise.map(files, function(file) {
- let filepath = path + "/" + file;
+ scanning++;
- return new Promise(function(resolve, reject) {
- fs.stat(filepath, function(err, stats) {
- if (err) {
- console.warn("Could not stat " + filepath);
- return resolve(false);
- }
+ let hasThumbs = false;
+ for (let i = 0; i < files.length; i++) {
+ if (files[i] == "thumbs") {
+ hasThumbs = true;
+ break;
+ }
+ }
- if (stats.isDirectory()) {
- return scanDir(filepath, stats).then(function(entry) {
- return resolve(true);
- });
- }
+ let tmp = Promise.resolve();
- /* stats.isFile() */
- return scanFile(path, file, stats).then(function(entry) {
- if (!entry) {
- return resolve(false);
+ if (!hasThumbs) {
+ tmp = new Promise(function(resolve, reject) {
+ fs.mkdir(path + "/thumbs", function(err) {
+ if (err) {
+ return reject("Unable to create " + paths + "/thumbs");
}
-
- const replacements = {
- path: path.slice(picturesPath.length),
- filename: file,
- added: new Date()
- };
-
- return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", {
- replacements: replacements,
- type: photoDB.sequelize.QueryTypes.SELECT
- }).then(function(photo) {
- if (photo.length == 0) {
- return photoDB.sequelize.query("INSERT INTO photos " +
- "SET path=:path,filename=:filename,added=DATE(:added)", {
- replacements: replacements
- });
- }
- }).then(function() {
- return resolve(true);
- });
+ return resolve();
});
});
- });
- }, {
- concurrency: 10
- }).then(function() {
- scanning--;
- if (scanning == 0) {
- const endStamp = Date.now();
- console.log("Scanning completed in " + Math.round(((endStamp - startStamp))) + "ms.");
}
- }).then(function() {
- return resolve();
+
+ return tmp.then(function() {
+ return Promise.map(files, function(file) {
+ let filepath = path + "/" + file;
+ if (file == "thumbs") {
+ return resolve(true);
+ }
+
+ return new Promise(function(resolve, reject) {
+ fs.stat(filepath, function(err, stats) {
+ if (err) {
+ console.warn("Could not stat " + filepath);
+ return resolve(false);
+ }
+
+ if (stats.isDirectory()) {
+ return scanDir(parent, filepath, stats).then(function(entry) {
+ return resolve(true);
+ });
+ }
+
+ /* stats.isFile() */
+ if (!re.exec(file)) {
+ return resolve(true);
+ }
+
+ const replacements = {
+ path: path.slice(picturesPath.length),
+ filename: file
+ };
+
+ return photoDB.sequelize.query("SELECT id FROM photos WHERE path=:path AND filename=:filename", {
+ replacements: replacements,
+ type: photoDB.sequelize.QueryTypes.SELECT
+ }).then(function(photo) {
+ if (photo.length == 0) {
+ processQueue.push([ replacements.path, file, stats.mtime, parent ]);
+ }
+ return resolve(true);
+ });
+ });
+ });
+ }, {
+ concurrency: 10
+ }).then(function() {
+ scanning--;
+ if (scanning == 0) {
+ const endStamp = Date.now();
+ console.log("Scanning completed in " + Math.round(((endStamp - startStamp))) + "ms. " + processQueue.length + " items to process.");
+ }
+ }).then(function() {
+ return resolve();
+ });
+ });
});
});
});
@@ -90,9 +131,86 @@ function scanDir(path) {
const startStamp = Date.now();
+let processRunning = false;
+
+const { spawn } = require('child_process');
+
+const sharp = require("sharp"), exif = require("exif-reader");
+
+function triggerWatcher() {
+ setTimeout(triggerWatcher, 1000);
+
+ if (!processRunning && processQueue.length) {
+ processRunning = true;
+ return Promise.map(processQueue, function(entry) {
+ var path = entry[0], file = entry[1], created = entry[2], albumId = entry[3],
+ src = picturesPath + path + "/" + file,
+ dst = picturesPath + path + "/thumbs/" + file,
+ image = sharp(src);
+// console.log("Processing " + src);
+ return image.metadata().then(function(metadata) {
+ if (metadata.exif) {
+ metadata.exif = exif(metadata.exif);
+ delete metadata.exif.thumbnail;
+ delete metadata.exif.image;
+ for (var key in metadata.exif.exif) {
+ if (Buffer.isBuffer(metadata.exif.exif[key])) {
+ metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
+ }
+ }
+ }
+
+ let replacements = {
+ albumId: albumId,
+ path: path,
+ filename: file,
+ width: metadata.width,
+ height: metadata.height,
+ added: moment().format().replace(/T.*/, "")
+ };
+
+ if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
+ metadata.exif.exif.DateTimeOriginal.setHours(0, 0, 0, 0);
+ metadata.exif.exif.DateTimeOriginal = metadata.exif.exif.DateTimeOriginal.toISOString().replace(/T.*/, "");
+// console.log(metadata.exif.exif.DateTimeOriginal);
+ replacements.taken = moment(metadata.exif.exif.DateTimeOriginal, "YYYY-MM-DD").format().replace(/T.*/, "");
+ replacements.modified = moment(metadata.exif.exif.DateTimeOriginal).format().replace(/T.*/, "");
+ } else {
+// console.log("Missing EXIF info for: " + file);
+ //console.log(JSON.stringify(metadata.exif, null, 2));
+ let patterns = /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]*)/, date = replacements.added;
+ let match = file.match(patterns);
+ if (match) {
+ date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
+// console.log("Constructed date: " + date);
+ } else {
+ date = moment(created).format();
+// console.log("Date from file: ", src, date);
+ }
+ replacements.taken = replacements.modified = date;
+ }
+
+ return image.resize(256, 256).toFile(dst).then(function() {
+ return photoDB.sequelize.query("INSERT INTO photos " +
+ "SET albumId=:albumId,path=:path,filename=:filename,added=DATE(:added),modified=DATE(:modified),taken=DATE(:taken),width=:width,height=:height", {
+ replacements: replacements
+ });
+ }).catch(function(error) {
+ console.log("Error resizing or writing " + src, error);
+ return Promise.Reject();
+ });
+ });
+ }, {
+ concurrency: 1
+ });
+ }
+}
+
module.exports = {
scan: function (db) {
photoDB = db;
- return scanDir(picturesPath);
+ return scanDir(0, picturesPath).then(function() {
+ triggerWatcher();
+ });
}
};