From 0d687cd2d0fbe3aacc370f94cf7dd69add577133 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Sun, 14 Oct 2018 21:28:45 -0700 Subject: [PATCH] Fix #11 - Update hash values when files are modified in rotation actions Signed-off-by: James Ketrenos --- server/routes/photos.js | 133 ++++++++++++++++++++++++++++++++++++---- server/scanner.js | 6 +- 2 files changed, 126 insertions(+), 13 deletions(-) diff --git a/server/routes/photos.js b/server/routes/photos.js index fb9f0d5..857761d 100755 --- a/server/routes/photos.js +++ b/server/routes/photos.js @@ -2,9 +2,12 @@ const express = require("express"), fs = require("fs"), - url = require("url"), config = require("config"), - moment = require("moment"); + moment = require("moment"), + crypto = require("crypto"), + util = require("util"); + +const execFile = util.promisify(require("child_process").execFile); let photoDB; @@ -54,6 +57,33 @@ const rename = function (_src, _dst) { } +const computeHash = function(_filepath) { + if (_filepath.indexOf(picturesPath.replace(/\/$/, "")) == 0) { + _filepath = _filepath.substring(picturesPath.length); + } + + let filepath = picturesPath + _filepath; + return new Promise(function(resolve, reject) { + let input = fs.createReadStream(filepath), + hash = crypto.createHash("sha256"); + if (!input) { + return reject(); + } + + input.on("readable", function() { + const data = input.read(); + if (data) { + hash.update(data); + } else { + input.close(); + resolve(hash.digest("hex")); + hash = null; + input = null; + } + }); + }); +}; + const stat = function (_path) { if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) { _path = _path.substring(picturesPath.length); @@ -71,7 +101,10 @@ const stat = function (_path) { }); } -const sharp = require("sharp"), exif = require("exif-reader"); +const sharp = require("sharp"); + +const inProcess = []; + router.put("/:id", function(req, res/*, next*/) { if (!req.user.maintainer) { @@ -82,6 +115,13 @@ router.put("/:id", function(req, res/*, next*/) { id: req.params.id }; + if (inProcess.indexOf(req.params.id) != -1) { + console.log("Request to modify asset currently in modification: " + req.params.id); + return res.status(409).send("Asset " + req.params.id + " is already being processed. Please try again."); + } + + inProcess.push(req.params.id); + console.log("PUT /" + replacements.id, req.query); switch (req.query.a) { case "undelete": @@ -90,6 +130,11 @@ router.put("/:id", function(req, res/*, next*/) { replacements: replacements }).then(function() { return res.status(200).send(req.params.id + " updated."); + }).catch(function(error) { + console.log(error); + return res.status(500).send(error); + }).then(function() { + inProcess.splice(inProcess.indexOf(req.params.id), 1); }); case "rename": @@ -108,6 +153,11 @@ router.put("/:id", function(req, res/*, next*/) { return res.status(200).send(asset); }); }); + }).catch(function(error) { + console.log(error); + return res.status(500).send(error); + }).then(function() { + inProcess.splice(inProcess.indexOf(req.params.id), 1); }); case "rotate": @@ -133,7 +183,14 @@ router.put("/:id", function(req, res/*, next*/) { asset.image = sharp(original); return asset.image.rotate(direction).withMetadata().toFile(target).then(function() { - /*...*/ + let stamp = moment(new Date(asset.modified)).format("YYYYMMDDhhmm.ss"); + console.log("Restamping " + target + " to " + stamp); + /* Re-stamp the file's ctime with the original ctime */ + return execFile("touch", [ + "-t", + stamp, + target + ]); }).then(function() { return asset.image.rotate(direction).resize(256, 256).withMetadata().toFile(thumb); }).then(function() { @@ -144,24 +201,76 @@ router.put("/:id", function(req, res/*, next*/) { throw "Unable to find original file after attempting to rotate!"; } asset.size = stats.size; - return photoDB.sequelize.query("UPDATE photos SET " + - "modified=CURRENT_TIMESTAMP,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " + - "WHERE id=:id", { - replacements: asset + asset.modified = stats.mtime; + return unlink(original).then(function() { + return rename(target, original); }).then(function() { - return unlink(original).then(function() { - return rename(target, original); + return photoDB.sequelize.query("UPDATE photos SET " + + "modified=:modified,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " + + "WHERE id=:id", { + replacements: asset }); }); }); }).then(function() { sharp.cache(false); sharp.cache(true); - return res.status(200).send(asset); + res.status(200).send(asset); + + return computeHash(asset.filepath).then(function(hash) { + asset.hash = hash; + return asset; + }).then(function(asset) { + return photoDB.sequelize.query("SELECT photos.id,photohashes.*,photos.filename,albums.path FROM photohashes " + + "LEFT JOIN photos ON (photos.id=photohashes.photoId) " + + "LEFT JOIN albums ON (albums.id=photos.albumId) " + + "WHERE hash=:hash OR photoId=:id", { + replacements: asset, + type: photoDB.sequelize.QueryTypes.SELECT + }).then(function(results) { + let query; + + if (results.length == 0) { + query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; + console.warn("HASH being updated and photoId " + asset.id + " *should* already exist, but it doesn't."); + } else if (results.length > 1) { + /* This image is now a duplicate! */ + for (var i = 0; i < results.length; i++) { + if (results[i].id != asset.id) { + console.log("Duplicate asset: " + + "'" + asset.filepath + "' is a copy of " + + "'" + results[i].path + results[i].filename + "'"); + asset.duplicate = results[i].id; + break; + } + } + + query = "UPDATE photos SET duplicate=:duplicate WHERE id=:id; " + + "DELETE FROM photohashes WHERE photoId=:id"; + + console.log("Updating photo " + asset.id + " as duplicate of " + asset.duplicate); + } else if (results[0].hash != asset.hash) { + query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id; UPDATE photos SET duplicate=0 WHERE id=:id"; + + console.log("Updating photohash for " + asset.id + " to " + asset.hash + ", and clearing duplicate field."); + } else { + console.log("Unexpected!", asset, results[0]); + return; + } + + return photoDB.sequelize.query(query, { + replacements: asset, + }).then(function() { + return asset; + }); + }); + }); }); }).catch(function(error) { console.log(error); return res.status(500).send(error); + }).then(function() { + inProcess.splice(inProcess.indexOf(req.params.id), 1); }); } @@ -170,7 +279,7 @@ router.put("/:id", function(req, res/*, next*/) { const getPhoto = function(id) { return photoDB.sequelize.query("SELECT " + - "photos.*,albums.path AS path,photohashes.hash,(albums.path || photos.filename) AS filepath FROM photos " + + "photos.*,albums.path AS path,photohashes.hash,modified,(albums.path || photos.filename) AS filepath FROM photos " + "LEFT JOIN albums ON albums.id=photos.albumId " + "LEFT JOIN photohashes ON photohashes.photoId=photos.id " + "WHERE photos.id=:id", { diff --git a/server/scanner.js b/server/scanner.js index a9d1a47..54135cd 100755 --- a/server/scanner.js +++ b/server/scanner.js @@ -233,6 +233,7 @@ function processBlock(items) { } /* Even if the hash doesn't need to be updated, the entry needs to be scanned */ + console.log("process needed because of " + query); needsProcessing.push(asset); if (!query) { @@ -616,7 +617,7 @@ function computeHash(filepath) { let input = fs.createReadStream(filepath), hash = crypto.createHash("sha256"); if (!input) { - return reject() + return reject(); } input.on("readable", function() { @@ -722,6 +723,9 @@ function doScan() { newEntries++; } if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) { + if (!asset.scanned) { console.log("no scan date on asset"); } + if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); } + if (!asset.modified) { console.log("no mtime."); } needsProcessing.push(asset); } else { updateScanned.push(asset.id);