From 44b157ee8b3f319199b3754d5ac913306f349712 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Fri, 3 Jan 2020 16:40:21 -0800 Subject: [PATCH] Switched from ufraw to darktable as ufraw is no longer in Ubuntu Signed-off-by: James Ketrenos --- Dockerfile | 10 +-- README.md | 8 +- server/scanner.js | 206 ++++++++++++++++++++-------------------------- 3 files changed, 98 insertions(+), 126 deletions(-) diff --git a/Dockerfile b/Dockerfile index 44039c3..f980156 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,8 @@ RUN DEBIAN_FRONTEND=NONINTERACTIVE apt-get install -y \ make \ cmake -# You can then install the latest npm, polymer-cli, and bower: -RUN npm install --global npm@latest npx +# You can then install the latest npm and npx +RUN npm install --global npm@latest # Speed up face-recognition and dev tools RUN apt-get install -y libopenblas-dev @@ -22,13 +22,13 @@ RUN apt-get install -y libopenblas-dev # Required for dlib to build RUN apt-get install -y libx11-dev libpng16-16 -# NEF processing uses ufraw-batch -RUN apt-get install -y ufraw-batch +# NEF processing uses darktable-cli, provided via darktable +RUN apt-get install -y darktable # Create a user with sudo access RUN DEBIAN_FRONTEND=noninteractive \ && apt-get install --no-install-recommends -y \ - sudo + sudo # NOTE: Requires 'sudo' package to already be installed RUN groupadd -g 1000 user \ diff --git a/README.md b/README.md index ab7ea08..2531654 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,8 @@ wget -qO- https://deb.nodesource.com/setup_8.x | sudo bash - sudo apt-get install --yes nodejs ``` -You can then install the latest npm, polymer-cli, and bower: - ```bash sudo npm install --global npm@latest -sudo npm install --global polymer-cli -sudo npm install --global bower ``` # Install BLAS to improve performance, and dev tools so @@ -40,9 +36,9 @@ sudo apt install -y libopenblas-dev cmake ``` ### -NEF processing uses ufraw-batch +NEF processing uses darktable ``` -sudo apt install -y ufraw-batch +sudo apt install -y darktable ``` ### Create `photos` user for DB diff --git a/server/scanner.js b/server/scanner.js index 8ab1946..3687872 100755 --- a/server/scanner.js +++ b/server/scanner.js @@ -61,8 +61,8 @@ const { spawn } = require('child_process'); const sharp = require("sharp"), exif = require("exif-reader"); -function convertRawToJpg(path, file) { - setStatus("Converting " + path + file); +function convertRawToJpg(path, raw, file) { + setStatus(`Converting ${path}${raw} to ${file}.`); path = picturesPath + path; @@ -73,30 +73,23 @@ function convertRawToJpg(path, file) { return; } - const ufraw = spawn("ufraw-batch", [ - "--silent", - "--wb=camera", - "--rotate=camera", - "--out-type=jpg", - "--compression=90", - "--exif", - "--overwrite", - "--output", path + file.replace(rawExtension, ".jpg"), + const darktable = spawn("darktable-cli", [ + path + raw, path + file ]); const stderr = []; - ufraw.stderr.on('data', function(data) { + darktable.stderr.on('data', function(data) { stderr.push(data); }); - return new Promise(function(ufraw, resolve, reject) { - ufraw.on('exit', function(stderr, code, signal) { + return new Promise((resolve, reject) => { + darktable.on('exit', (code, signal) => { if (signal || code != 0) { - let error = "UFRAW for " + path + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n"; + let error = "darktable for " + path + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n"; setStatus(error, "error"); return moveCorrupt(path, file).then(function() { - setStatus("ufraw failed", "warn"); + setStatus("darktable failed", "warn"); return reject(error); }).catch(function(error) { setStatus("moveCorrupt failed", "warn"); @@ -104,9 +97,9 @@ function convertRawToJpg(path, file) { }); } return mkdir(path + "raw").then(function() { - fs.rename(path + file, path + "raw/" + file, function(err) { + fs.rename(path + raw, path + "raw/" + raw, function(err) { if (err) { - setStatus("Unable to move RAW file: " + path + file, "error"); + setStatus("Unable to move RAW file: " + path + raw, "error"); return reject(err); } return resolve(); @@ -115,8 +108,8 @@ function convertRawToJpg(path, file) { setStatus("mkdir failed", "warn"); return reject(error); }); - }.bind(this, ufraw)); - }.bind(this, stderr)); + }); + }); }); }); } @@ -164,76 +157,78 @@ function processBlock(items) { let toProcess = processing.length, lastMessage = moment(); setStatus("Items to be processed: " + toProcess); - return Promise.mapSeries(processing, function(asset) { - return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) { - asset.hash = hash; - return asset; - }).then(function(asset) { - return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " + - "LEFT JOIN photos ON (photos.id=photohashes.photoId) " + - "LEFT JOIN albums ON (albums.id=photos.albumId) " + - "WHERE hash=:hash OR photoId=:id", { - replacements: asset, - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(results) { - let query; + return Promise.mapSeries(processing, (asset) => { + if (!asset.raw) { + return; + } + + const path = asset.album.path; - if (results.length == 0) { - query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; - } else if (results[0].hash != asset.hash) { - query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id"; - } else if (results[0].photoId != asset.id) { - setStatus("Duplicate asset: " + - "'" + asset.album.path + asset.filename + "' is a copy of " + - "'" + results[0].path + results[0].filename + "'"); - if (asset.duplicate != results[0].photoId) { - asset.duplicate = results[0].photoId; - duplicates.push(asset); - } - return null; - } - - /* Even if the hash doesn't need to be updated, the entry needs to be scanned */ -// console.log("process needed because of " + query); - needsProcessing.push(asset); - - if (!query) { - return asset; - } - - return photoDB.sequelize.query(query, { - replacements: asset, - }).then(function() { - return asset; - }); - }); - }).then(function(asset) { - if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */ - return; + return exists(picturesPath + path + asset.filename).then(function(exist) { + if (exist) { + return asset; } + + return mkdir(picturesPath + path + "raw").then(function() { + return convertRawToJpg(path, asset.raw, asset.filename); + }).then(function() { + console.log("Done converting..."); + }); + }); + }).then(() => { + return Promise.mapSeries(processing, (asset) => { + return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) { + asset.hash = hash; + return asset; + }).then(function(asset) { + return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " + + "LEFT JOIN photos ON (photos.id=photohashes.photoId) " + + "LEFT JOIN albums ON (albums.id=photos.albumId) " + + "WHERE hash=:hash OR photoId=:id", { + replacements: asset, + type: photoDB.sequelize.QueryTypes.SELECT + }).then(function(results) { + let query; - var path = asset.album.path, - file = asset.filename, - created = asset.stats.mtime, - albumId = asset.album.id; - - let tmp = Promise.resolve(file); - /* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */ - if (rawExtension.exec(file)) { - tmp = exists(picturesPath + path + file.replace(rawExtension, ".jpg")).then(function(exist) { - if (exist) { - return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */ + if (results.length == 0) { + query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; + } else if (results[0].hash != asset.hash) { + query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id"; + } else if (results[0].photoId != asset.id) { + setStatus("Duplicate asset: " + + "'" + asset.album.path + asset.filename + "' is a copy of " + + "'" + results[0].path + results[0].filename + "'"); + if (asset.duplicate != results[0].photoId) { + asset.duplicate = results[0].photoId; + duplicates.push(asset); + } + return null; } - - return mkdir(picturesPath + path + "raw").then(function() { - return convertRawToJpg(path, file); + + /* Even if the hash doesn't need to be updated, the entry needs to be scanned */ + // console.log("process needed because of " + query); + needsProcessing.push(asset); + + if (!query) { + return asset; + } + + return photoDB.sequelize.query(query, { + replacements: asset, }).then(function() { - return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */ + return asset; }); }); - } + }).then(function(asset) { + if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */ + return; + } - return tmp.then(function(file) { + var path = asset.album.path, + file = asset.filename, + created = asset.stats.mtime, + albumId = asset.album.id; + var src = picturesPath + path + file, image = sharp(src); @@ -321,38 +316,15 @@ function processBlock(items) { }); }).catch(function(error) { setStatus("Error reading image " + src + ":\n" + error, "error"); - return moveCorrupt(path, file).then(function() { - - /* If the original file was not a NEF/ORF, we are done... */ - if (!rawExtension.exec(asset.filename)) { - return; - } - - /* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */ - for (var i = 0; i < triedClean.length; i++) { - if (triedClean[i] == path + file) { - /* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */ - // return moveCorrupt(path, asset.filename); - - setStatus("Already attempted to convert NEF/ORF to JPG: " + path + file, "error"); - return; - } - } - - setStatus("Adding " + path + file + " back onto processing queue.", "error"); - triedClean.push(path + file); - processBlock([ path, file, created, albumId ]); - }); + return moveCorrupt(path, file); }); - }).catch(function() { - setStatus("Continuing file processing.", "warn"); + }).then(function() { + toProcess--; + if (moment().add(-5, 'seconds') > lastMessage) { + setStatus("Items to be processed: " + toProcess); + lastMessage = moment(); + } }); - }).then(function() { - toProcess--; - if (moment().add(-5, 'seconds') > lastMessage) { - setStatus("Items to be processed: " + toProcess); - lastMessage = moment(); - } }); }).catch(function(error) { setStatus("Error processing file. Continuing.", "error"); @@ -469,8 +441,8 @@ function scanDir(parent, path) { album.hasAssets = true; - assets.push({ - filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */ + const asset = { + filename: file.replace(rawExtension, ".jpg"), name: file.replace(/.[^.]*$/, ""), stats: { mtime: stats.mtime, @@ -478,7 +450,11 @@ function scanDir(parent, path) { }, size: stats.size, album: album - }); + } + if (file != asset.filename) { + asset.raw = file; + } + assets.push(asset); }); }); }).then(function() {