From 744e2535bcd19883dc0b9c329185253917c488ce Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Wed, 24 Oct 2018 20:44:08 -0700 Subject: [PATCH] Changed 'DELETE' action to be a long-running task Signed-off-by: James Ketrenos --- frontend/src/ketr-photos/ketr-photos.html | 68 +++++- server/db/users.js | 4 +- server/routes/photos.js | 269 ++++++++++++++-------- 3 files changed, 239 insertions(+), 102 deletions(-) diff --git a/frontend/src/ketr-photos/ketr-photos.html b/frontend/src/ketr-photos/ketr-photos.html index 1092da1..b925b1b 100755 --- a/frontend/src/ketr-photos/ketr-photos.html +++ b/frontend/src/ketr-photos/ketr-photos.html @@ -1427,19 +1427,83 @@ }.bind(this, thumbnail), {}, "PUT"); }, + purgeStatus: function() { + console.log("Checking purge status", this.purgeTask); + window.fetch("api/v1/photos/status/" + this.purgeTask.token, function(error, xhr) { + if (error && xhr.status != 404) { + console.log("Unable to take action on photo: " + error); + return; + } + + if (xhr.status == 404) { + console.log("Task complete"); + this.purgeTask = null; + this.resetPhotos(); + return; + } + + var results; + try { + results = JSON.parse(xhr.responseText); + } catch (___) { + this.$.toast.text = "Unable to parse delete task."; + this.$.toast.setAttribute("error", true); + this.$.toast.updateStyles(); + this.$.toast.show(); + console.error("Unable to parse photos"); + return; + } + + if (results.remaining == 0) { + console.log("Task complete"); + this.purgeTask = null; + this.resetPhotos(); + } else { + this.purgeTask = results; + if (this.purgeTask.eta == -1) { + this.purgeTask.eta = 1000; + } + console.log("Task ETA: " + this.purgeTask.eta); + this.async(function() { + this.purgeStatus(); + this.resetPhotos(); + }, this.purgeTask.eta || 1000); + } + }.bind(this)); + }, + purgeTrashAction: function() { var query = ""; if (this.mode == "trash") { console.log("TODO: Prompt user 'Are you sure?' ?"); query = "?permanent=1"; } - window.fetch("api/v1/photos/" + query, function(error) { + window.fetch("api/v1/photos/" + query, function(error, xhr) { if (error) { console.log("Unable to take action on photo: " + error); return; } - this.resetPhotos(); + var results; + try { + results = JSON.parse(xhr.responseText); + } catch (___) { + this.$.toast.text = "Unable to parse delete task."; + this.$.toast.setAttribute("error", true); + this.$.toast.updateStyles(); + this.$.toast.show(); + console.error("Unable to parse photos"); + return; + } + + if (results.task) { + this.purgeTask = results.task; + this.async(function() { + this.purgeStatus(); + }, 250); + } else { + this.resetPhotos(); + } }.bind(this), {}, "DELETE"); }, diff --git a/server/db/users.js b/server/db/users.js index 74a109e..6e9555b 100644 --- a/server/db/users.js +++ b/server/db/users.js @@ -13,9 +13,7 @@ * to create db connections, test the connection, then create the tables and * relationships if not present */ -const fs = require('fs'), - path = require('path'), - Sequelize = require('sequelize'), +const Sequelize = require('sequelize'), config = require('config'); function init() { diff --git a/server/routes/photos.js b/server/routes/photos.js index beedec3..163fbeb 100755 --- a/server/routes/photos.js +++ b/server/routes/photos.js @@ -315,6 +315,134 @@ const getPhoto = function(id) { }); } +const tasks = []; +const Task = function() { + return new Promise(function(resolve, reject) { + crypto.randomBytes(16, function(error, buffer) { + if (error) { + return reject(error); + } + + let now = Date.now(), task; + + task = { + started: now, + lastUpdate: now, + eta: now, + processed: 0, + remaining: 0, + token: buffer.toString('hex'), + }; + + tasks.push(task); + return resolve(task); + }); + }); +}; + +const removeTask = function(task) { + let i = tasks.indexOf(task); + if (i != -1) { + tasks.splice(i, 1); + } +} + +router.get("/status/:token", function(req, res) { + if (!req.params.token) { + return res.status(400).send("Usage /status/:token"); + } + + for (var i = 0; i < tasks.length; i++) { + if (tasks[i].token == req.params.token) { + return res.status(200).send(tasks[i]); + } + } + + return res.status(404).send("Task " + req.params.token + " not found."); +}); + +/** + * 1. Look if there are duplicates. + * 2. Update all other duplicates to be duplicates of the first image that remains + * 3. If no duplicates, DELETE the entry from photohashes + * 4. If there are duplicates, update the HASH entry to point to the first image that remains + * 5. Delete the entry from photos + * 6. Delete the scaled, thumb, and original from disk + */ +const deletePhoto = function(photo) { + return photoDB.sequelize.transaction(function(transaction) { + return photoDB.sequelize.query("SELECT id FROM photos WHERE duplicate=:id", { + replacements: photo, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }).then(function(duplicates) { + if (!duplicates.length) { + return null; + } + + let first = duplicates.shift(); + let needsUpdate = []; + duplicates.forEach(function(duplicate) { + needsUpdate.push(duplicate.id); + }); + + if (!needsUpdate.length) { + return first; + } + + // 2. Update all other duplicates to be duplicates of the first image that remains + console.log("Updating " + needsUpdate + " to point to " + first.id); + return photoDB.sequelize.query( + "UPDATE photos SET duplicate=:first WHERE id IN (:needsUpdate)", { + replacements: { + first: first.id, + needsUpdate: needsUpdate + }, + transaction: transaction + }).then(function() { + return first; + }); + }).then(function(first) { + if (!first) { + console.log("Deleting "+ photo.id + " from photohash."); + // 3. If no duplicates, DELETE the entry from photohashes + return photoDB.sequelize.query( + "DELETE FROM photohashes WHERE photoId=:id", { + replacements: photo, + transaction: transaction + }); + } + console.log("Updating photohash for " + photo.id + " to point to " + first.id); + // 4. If there are duplicates, update the HASH entry to point to the first image that remains + return photoDB.sequelize.query( + "UPDATE photohashes SET photoId=:first WHERE photoId=:photo", { + replacements: { + first: first.id, + photo: photo.id + }, + transaction: transaction + }); + }).then(function() { + console.log("Deleting " + photo.path + photo.filename + " from DB."); + // 5. Delete the entry from photos + return photoDB.sequelize.query("DELETE FROM photos WHERE id=:id", { + replacements: photo, + transaction: transaction + }); + }).then(function() { + // 6. Delete the scaled, thumb, and original from disk + console.log("Deleting " + photo.path + photo.filename + " from disk."); + return unlink(photo.path + "thumbs/scaled/" + photo.filename).catch(function() {}).then(function() { + return unlink(photo.path + "thumbs/" + photo.filename).catch(function() {}).then(function() { + return unlink(photo.path + photo.filename).catch(function(error) { + console.log("Error removing file: " + error); + }); + }); + }); + }); + }); +} + router.delete("/:id?", function(req, res/*, next*/) { if (!req.user.maintainer) { return res.status(401).send("Unauthorized to delete photos."); @@ -340,6 +468,8 @@ router.delete("/:id?", function(req, res/*, next*/) { console.log("DELETE /" + replacements.id, req.query); + let sent = false; + return photoDB.sequelize.query("SELECT " + "photos.*,albums.path AS path,photohashes.hash,(albums.path || photos.filename) AS filepath FROM photos " + "LEFT JOIN albums ON albums.id=photos.albumId " + @@ -350,115 +480,61 @@ router.delete("/:id?", function(req, res/*, next*/) { raw: true }).then(function(photos) { if (photos.length == 0) { - res.status(404).send("Unable to find photo " + replacements.id); - return true; + sent = true; + return res.status(404).send("Unable to find photo " + replacements.id); } if (!req.query.permanent) { return photoDB.sequelize.query("UPDATE photos SET deleted=1,updated=CURRENT_TIMESTAMP WHERE id=:id", { replacements: replacements }).then(function() { - return false; - }); - } - - /** - * Delete the asset from disk and the DB - * 1. Look if there are duplicates. - * 2. Update all other duplicates to be duplicates of the first image that remains - * 3. If no duplicates, DELETE the entry from photohashes - * 4. If there are duplicates, update the HASH entry to point to the first image that remains - * 5. Delete the entry from photos - * 6. Delete the scaled, thumb, and original from disk - */ - return Promise.mapSeries(photos, function(photo) { - return photoDB.sequelize.transaction(function(transaction) { - return photoDB.sequelize.query("SELECT id FROM photos WHERE duplicate=:id", { - replacements: photo, - type: photoDB.Sequelize.QueryTypes.SELECT, - raw: true - }).then(function(duplicates) { - if (!duplicates.length) { - return null; - } - - let first = duplicates.shift(); - let needsUpdate = []; - duplicates.forEach(function(duplicate) { - needsUpdate.push(duplicate.id); - }); - - if (!needsUpdate.length) { - return first; - } - - // 2. Update all other duplicates to be duplicates of the first image that remains - console.log("Updating " + needsUpdate + " to point to " + first.id); - return photoDB.sequelize.query( - "UPDATE photos SET duplicate=:first WHERE id IN (:needsUpdate)", { - replacements: { - first: first.id, - needsUpdate: needsUpdate - }, - transaction: transaction - }).then(function() { - return first; - }); - }).then(function(first) { - if (!first) { - console.log("Deleting "+ photo.id + " from photohash."); - // 3. If no duplicates, DELETE the entry from photohashes - return photoDB.sequelize.query( - "DELETE FROM photohashes WHERE photoId=:id", { - replacements: photo, - transaction: transaction - }); - } - console.log("Updating photohash for " + photo.id + " to point to " + first.id); - // 4. If there are duplicates, update the HASH entry to point to the first image that remains - return photoDB.sequelize.query( - "UPDATE photohashes SET photoId=:first WHERE photoId=:photo", { - replacements: { - first: first.id, - photo: photo.id - }, - transaction: transaction - }); - }).then(function() { - console.log("Deleting " + photo.path + photo.filename + " from DB."); - // 5. Delete the entry from photos - return photoDB.sequelize.query("DELETE FROM photos WHERE id=:id", { - replacements: photo, - transaction: transaction - }); - }).then(function() { - // 6. Delete the scaled, thumb, and original from disk - console.log("Deleting " + photo.path + photo.filename + " from disk."); - return unlink(photo.path + "thumbs/scaled/" + photo.filename).catch(function() {}).then(function() { - return unlink(photo.path + "thumbs/" + photo.filename).catch(function() {}).then(function() { - return unlink(photo.path + photo.filename).catch(function(error) { - console.log("Error removing file: " + error); - }); - }); - }); + sent = true; + return res.status(200).send({ + id: req.params.id, + deleted: true, + permanent: (req.query && req.query.permanent) || false }); }); - }).then(function() { - return false; - }); - }).then(function(sent) { - if (sent) { - return; } + + /** + * Delete the asset from disk and the DB + */ + return Task().then(function(task) { + task.remaining = photos.length; + task.eta = -1; - return res.status(200).send({ - id: req.params.id, - deleted: true, - permanent: (req.query && req.query.permanent) || false + sent = true; + res.status(200).send({ + id: req.params.id, + task: task, + permanent: (req.query && req.query.permanent) || false + }); + + return Promise.mapSeries(photos, function(photo) { + let lastStamp = Date.now(); + return deletePhoto(photo).then(function() { + let now = Date.now(), elapsed = now - lastStamp; + lastStamp= now; + task.processed++; + task.remaining--; + task.lastUpdate = Date.now(); + task.eta = elapsed * task.remaining; + }) + }).then(function() { + console.log("Processing task " + task.token + " finished: " + (task.lastUpdate - task.started)); + removeTask(task); + }).catch(function(error) { + console.log("Processing task " + task.token + " failed: " + error); + removeTask(task); + }); }); }).catch(function(error) { console.log(error); - return res.status(500).send("Unable to delete photo " + req.params.id + ": " + error); + if (!sent) { + sent = true; + res.status(500).send("Unable to delete photo " + req.params.id + ": " + error); + } }); }); @@ -662,5 +738,4 @@ router.get("/*", function(req, res/*, next*/) { }); }); - module.exports = router;