From 4a9bb061b069b2cd60e1c7b85a40037386173d5b Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Fri, 28 Sep 2018 01:37:44 -0700 Subject: [PATCH] Fixed EMFILE problem Signed-off-by: James Ketrenos --- frontend/src/ketr-photos/ketr-photos.html | 18 ++++--- server/db/users.js | 1 + server/routes/users.js | 19 +++---- server/scanner.js | 61 +++++++++++++---------- 4 files changed, 56 insertions(+), 43 deletions(-) diff --git a/frontend/src/ketr-photos/ketr-photos.html b/frontend/src/ketr-photos/ketr-photos.html index 89928d7..a02f78a 100755 --- a/frontend/src/ketr-photos/ketr-photos.html +++ b/frontend/src/ketr-photos/ketr-photos.html @@ -258,7 +258,10 @@ font-weight: bold; } - #memories [is-today] paper-icon-button { + #requestAccess { + max-width: 60ex; + border: 1px solid #444; + box-sizing: border-box; } #requestAccess div > div { @@ -372,15 +375,16 @@
-
Hello!
+
Create an account
- Unfortunately, I haven't built this part of the site yet... send me an email (james @ ketrenos.com) - and I'll create an account for you. +

To have your account activated, tell me who you know in the 'who do you know?' field.

+

Thanks,

+

James

- + - +
create
@@ -600,7 +604,7 @@ this.user = user; } }.bind(this), null, "POST", { - u: this.username, + w: this.who, p: this.password, n: this.name, m: this.mail diff --git a/server/db/users.js b/server/db/users.js index e8baedb..342970f 100644 --- a/server/db/users.js +++ b/server/db/users.js @@ -32,6 +32,7 @@ function init() { autoIncrement: true }, displayName: Sequelize.STRING, + notes: Sequelize.STRING, uid: Sequelize.STRING, isLDAP: Sequelize.BOOLEAN, authToken: Sequelize.STRING, diff --git a/server/routes/users.js b/server/routes/users.js index 2f86171..346d912 100755 --- a/server/routes/users.js +++ b/server/routes/users.js @@ -43,34 +43,35 @@ function ldapPromise(username, password) { } router.post("/create", function(req, res) { - let username = req.query.u || req.body.u || "", + let who = req.query.w || req.body.w || "", password = req.query.p || req.body.p || "", name = req.query.n || req.body.n || username, mail = req.query.m || req.body.m; - if (!username || !password || !mail || !name) { - return res.status(400).send("Missing user id, name, password, and/or email"); + if (!who || !password || !mail || !name) { + return res.status(400).send("Missing who you know, name, password, and/or email"); } let query = "SELECT * FROM users WHERE uid=:username"; return userDB.sequelize.query(query, { replacements: { - username: username + username: mail }, type: userDB.Sequelize.QueryTypes.SELECT }).then(function(results) { if (results.length != 0) { - return res.status(400).send("Username already exists."); + return res.status(400).send("Email address already used."); } return userDB.sequelize.query("INSERT INTO users " + - "(uid,displayName,password,mail,memberSince,authenticated) " + - "VALUES(:username,:name,:password,:mail,CURRENT_TIMESTAMP,0)", { + "(uid,displayName,password,mail,memberSince,authenticated,notes) " + + "VALUES(:username,:name,:password,:mail,CURRENT_TIMESTAMP,0,:notes)", { replacements: { - username: username, + username: mail, name: name, password: crypto.createHash('sha256').update(password).digest('base64'), - mail: mail + mail: mail, + notes: who } }).then(function(results) { /* diff --git a/server/scanner.js b/server/scanner.js index ca794eb..8d39d6a 100755 --- a/server/scanner.js +++ b/server/scanner.js @@ -7,8 +7,6 @@ const Promise = require("bluebird"), crypto = require("crypto"); -let scanning = 0; - let photoDB = null; const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/"; @@ -39,9 +37,6 @@ function removeNewerFile(path, fileA, fileB) { }); } - -const startStamp = Date.now(); - let processRunning = false; const { spawn } = require('child_process'); @@ -185,7 +180,6 @@ function moveCorrupt(path, file) { }); } -/*******************************************************************************************************/ function processBlock(items) { if (items) { @@ -207,10 +201,19 @@ function processBlock(items) { return a.stats.mtime - b.stats.mtime; }); + let toProcess = processing.length, lastMessage = moment(); return Promise.map(processing, function(asset) { return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) { asset.hash = hash; + }).then(function() { + toProcess--; + if (moment().add(-5, 'seconds') > lastMessage) { + console.log("Hash items to be processed: " + toProcess); + lastMessage = moment(); + } }); +}, { + concurrency: 5 }).then(function() { let toProcess = processing.length, lastMessage = moment(); /* Needs to be one at a time in case there are multiple HASH collisions */ @@ -248,7 +251,7 @@ function processBlock(items) { }).then(function() { toProcess--; if (moment().add(-5, 'seconds') > lastMessage) { - console.log("Hash items to be processed: " + toProcess); + console.log("Hash items to be checked: " + toProcess); lastMessage = moment(); } }); @@ -526,7 +529,10 @@ function scanDir(parent, path) { assets.push({ filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */ name: file.replace(/.[^.]*$/, ""), - stats: stats, + stats: { + ctime: stats.ctime, + mtime: stats.mtime + }, album: album }); }); @@ -611,25 +617,23 @@ function findOrUpdateDBAsset(asset) { function computeHash(filepath) { return new Promise(function(resolve, reject) { - try { - const input = fs.createReadStream(filepath), - hash = crypto.createHash("sha256"); - if (!input) { - return reject() - } - - input.on("readable", function() { - const data = input.read(); - if (data) { - hash.update(data); - } else { - resolve(hash.digest("hex")); - } - }); - } catch(error) { - console.error("Unable to create hash: " + error); - return reject(error); + let input = fs.createReadStream(filepath), + hash = crypto.createHash("sha256"); + if (!input) { + return reject() } + + input.on("readable", function() { + const data = input.read(); + if (data) { + hash.update(data); + } else { + input.close(); + resolve(hash.digest("hex")); + hash = null; + input = null; + } + }); }); } @@ -653,7 +657,7 @@ module.exports = { */ let initialized = Date.now(); let now = Date.now(); - const needsProcessing = []; + let needsProcessing = []; lastScan = new Date(); return scanDir(null, picturesPath).spread(function(albums, assets) { @@ -709,14 +713,17 @@ module.exports = { } }).then(function() { console.log("Updated scan date of " + updateScanned.length + " assets"); + updateScanned = []; }); } }).then(function() { console.log(needsProcessing.length + " assets need HASH computed"); processBlock(needsProcessing); + needsProcessing = []; }).then(function() { console.log("Scanned " + assets.length + " asset DB entries in " + ((Date.now() - now) / 1000) + "s"); + assets = []; }); }); }).then(function() {