Switched from ufraw to darktable as ufraw is no longer in Ubuntu

Signed-off-by: James Ketrenos <james_gitlab@ketrenos.com>

Dockerfile

@@ -13,8 +13,8 @@ RUN DEBIAN_FRONTEND=NONINTERACTIVE apt-get install -y \
   make \
   cmake
 
-# You can then install the latest npm, polymer-cli, and bower:
+# You can then install the latest npm and npx
-RUN npm install --global npm@latest npx
+RUN npm install --global npm@latest
 
 # Speed up face-recognition and dev tools
 RUN apt-get install -y libopenblas-dev
@@ -22,13 +22,13 @@ RUN apt-get install -y libopenblas-dev
 # Required for dlib to build
 RUN apt-get install -y libx11-dev libpng16-16
 
-# NEF processing uses ufraw-batch
+# NEF processing uses darktable-cli, provided via darktable
-RUN apt-get install -y ufraw-batch
+RUN apt-get install -y darktable
 
 # Create a user with sudo access
 RUN DEBIAN_FRONTEND=noninteractive \
   && apt-get install --no-install-recommends -y \
-      sudo
+    sudo
   
 # NOTE: Requires 'sudo' package to already be installed
 RUN groupadd -g 1000 user \

README.md

@@ -24,12 +24,8 @@ wget -qO- https://deb.nodesource.com/setup_8.x | sudo bash -
 sudo apt-get install --yes nodejs
 ```
 
-You can then install the latest npm, polymer-cli, and bower:
-
 ```bash
 sudo npm install --global npm@latest
-sudo npm install --global polymer-cli
-sudo npm install --global bower
 ```
 
 # Install BLAS to improve performance, and dev tools so
@@ -40,9 +36,9 @@ sudo apt install -y libopenblas-dev cmake
 ```
 
 ###
-NEF processing uses ufraw-batch
+NEF processing uses darktable
 ```
-sudo apt install -y ufraw-batch
+sudo apt install -y darktable
 ```
 
 ### Create `photos` user for DB

server/scanner.js

@@ -61,8 +61,8 @@ const { spawn } = require('child_process');
 
 const sharp = require("sharp"), exif = require("exif-reader");
 
-function convertRawToJpg(path, file) {
+function convertRawToJpg(path, raw, file) {
-  setStatus("Converting " + path + file);
+  setStatus(`Converting ${path}${raw} to ${file}.`);
 
   path = picturesPath + path;
 
@@ -73,30 +73,23 @@ function convertRawToJpg(path, file) {
         return;
       }
 
-      const ufraw = spawn("ufraw-batch", [
+      const darktable = spawn("darktable-cli", [
-        "--silent",
+        path + raw,
-        "--wb=camera",
-        "--rotate=camera",
-        "--out-type=jpg",
-        "--compression=90",
-        "--exif",
-        "--overwrite",
-        "--output", path + file.replace(rawExtension, ".jpg"),
         path + file
       ]);
 
       const stderr = [];
-      ufraw.stderr.on('data', function(data) {
+      darktable.stderr.on('data', function(data) {
         stderr.push(data);
       });
 
-      return new Promise(function(ufraw, resolve, reject) {
+      return new Promise((resolve, reject) => {
-        ufraw.on('exit', function(stderr, code, signal) {
+        darktable.on('exit', (code, signal) => {
           if (signal || code != 0) {
-            let error = "UFRAW for " + path + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n";
+            let error = "darktable for " + path + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n";
             setStatus(error, "error");
             return moveCorrupt(path, file).then(function() {
-              setStatus("ufraw failed", "warn");
+              setStatus("darktable failed", "warn");
               return reject(error);
             }).catch(function(error) {
               setStatus("moveCorrupt failed", "warn");
@@ -104,9 +97,9 @@ function convertRawToJpg(path, file) {
             });
           }
           return mkdir(path + "raw").then(function() {
-            fs.rename(path + file, path + "raw/" + file, function(err) {
+            fs.rename(path + raw, path + "raw/" + raw, function(err) {
               if (err) {
-                setStatus("Unable to move RAW file: " + path + file, "error");
+                setStatus("Unable to move RAW file: " + path + raw, "error");
                 return reject(err);
               }
               return resolve();
@@ -115,8 +108,8 @@ function convertRawToJpg(path, file) {
             setStatus("mkdir failed", "warn");
             return reject(error);
           });
-        }.bind(this, ufraw));
+        });
-      }.bind(this, stderr));
+      });
     });
   });
 }
@@ -164,76 +157,78 @@ function processBlock(items) {
 
   let toProcess = processing.length, lastMessage = moment();
   setStatus("Items to be processed: "  + toProcess);
-  return Promise.mapSeries(processing, function(asset) {
+  return Promise.mapSeries(processing, (asset) => {
-    return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
+    if (!asset.raw) {
-      asset.hash = hash;
+      return;
-      return asset;
+    }
-    }).then(function(asset) {
+    
-      return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
+    const path = asset.album.path;
-        "LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
-        "LEFT JOIN albums ON (albums.id=photos.albumId) " +
-        "WHERE hash=:hash OR photoId=:id", {
-        replacements: asset,
-        type: photoDB.sequelize.QueryTypes.SELECT
-      }).then(function(results) {
-        let query;
 
-        if (results.length == 0) {
+    return exists(picturesPath + path + asset.filename).then(function(exist) {
-          query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
+      if (exist) {
-        } else if (results[0].hash != asset.hash) {
+        return asset;
-          query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
-        } else if (results[0].photoId != asset.id) {
-          setStatus("Duplicate asset: " + 
-            "'" + asset.album.path + asset.filename + "' is a copy of " +
-            "'" + results[0].path + results[0].filename + "'");
-          if (asset.duplicate != results[0].photoId) {
-            asset.duplicate = results[0].photoId;
-            duplicates.push(asset);
-          }
-          return null;
-        }
-
-        /* Even if the hash doesn't need to be updated, the entry needs to be scanned */
-//        console.log("process needed because of " + query);
-        needsProcessing.push(asset);
-
-        if (!query) {
-          return asset;
-        }
-
-        return photoDB.sequelize.query(query, {
-          replacements: asset,
-        }).then(function() {
-          return asset;
-        });
-      });
-    }).then(function(asset) {
-      if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */
-        return;
       }
+      
+      return mkdir(picturesPath + path + "raw").then(function() {
+        return convertRawToJpg(path, asset.raw, asset.filename);
+      }).then(function() {
+        console.log("Done converting...");
+      });
+    });
+  }).then(() => {
+    return Promise.mapSeries(processing, (asset) => {
+      return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
+        asset.hash = hash;
+        return asset;
+      }).then(function(asset) {
+        return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
+          "LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
+          "LEFT JOIN albums ON (albums.id=photos.albumId) " +
+          "WHERE hash=:hash OR photoId=:id", {
+          replacements: asset,
+          type: photoDB.sequelize.QueryTypes.SELECT
+        }).then(function(results) {
+          let query;
 
-      var path = asset.album.path,
+          if (results.length == 0) {
-        file = asset.filename,
+            query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
-        created = asset.stats.mtime,
+          } else if (results[0].hash != asset.hash) {
-        albumId = asset.album.id;
+            query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
-        
+          } else if (results[0].photoId != asset.id) {
-      let tmp = Promise.resolve(file);
+            setStatus("Duplicate asset: " + 
-      /* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */
+              "'" + asset.album.path + asset.filename + "' is a copy of " +
-      if (rawExtension.exec(file)) {
+              "'" + results[0].path + results[0].filename + "'");
-        tmp = exists(picturesPath + path + file.replace(rawExtension, ".jpg")).then(function(exist) {
+            if (asset.duplicate != results[0].photoId) {
-          if (exist) {
+              asset.duplicate = results[0].photoId;
-            return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
+              duplicates.push(asset);
+            }
+            return null;
           }
-          
+
-          return mkdir(picturesPath + path + "raw").then(function() {
+          /* Even if the hash doesn't need to be updated, the entry needs to be scanned */
-            return convertRawToJpg(path, file);
+  //        console.log("process needed because of " + query);
+          needsProcessing.push(asset);
+
+          if (!query) {
+            return asset;
+          }
+
+          return photoDB.sequelize.query(query, {
+            replacements: asset,
           }).then(function() {
-            return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
+            return asset;
           });
         });
-      }
+      }).then(function(asset) {
+        if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */
+          return;
+        }
 
-      return tmp.then(function(file) {
+        var path = asset.album.path,
+          file = asset.filename,
+          created = asset.stats.mtime,
+          albumId = asset.album.id;
+          
         var src = picturesPath + path + file,
           image = sharp(src);
 
@@ -321,38 +316,15 @@ function processBlock(items) {
           });
         }).catch(function(error) {
           setStatus("Error reading image " + src + ":\n" + error, "error");
-          return moveCorrupt(path, file).then(function() {
+          return moveCorrupt(path, file);
-
-            /* If the original file was not a NEF/ORF, we are done... */
-            if (!rawExtension.exec(asset.filename)) {
-              return;
-            }
-
-            /* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */
-            for (var i = 0; i < triedClean.length; i++) {
-              if (triedClean[i] == path + file) {
-                /* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */
-                // return moveCorrupt(path, asset.filename);
-
-                setStatus("Already attempted to convert NEF/ORF to JPG: " + path + file, "error");
-                return;
-              }
-            }
-
-            setStatus("Adding " + path + file + " back onto processing queue.", "error");
-            triedClean.push(path + file);
-            processBlock([ path, file, created, albumId ]);
-          });
         });
-      }).catch(function() {
+      }).then(function() {
-        setStatus("Continuing file processing.", "warn");
+        toProcess--;
+        if (moment().add(-5, 'seconds') > lastMessage) {
+          setStatus("Items to be processed: "  + toProcess);
+          lastMessage = moment();
+        }
       });
-    }).then(function() {
-      toProcess--;
-      if (moment().add(-5, 'seconds') > lastMessage) {
-        setStatus("Items to be processed: "  + toProcess);
-        lastMessage = moment();
-      }
     });
   }).catch(function(error) {
     setStatus("Error processing file. Continuing.", "error");
@@ -469,8 +441,8 @@ function scanDir(parent, path) {
 
         album.hasAssets = true;
 
-        assets.push({
+        const asset = {
-          filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
+          filename: file.replace(rawExtension, ".jpg"),
           name: file.replace(/.[^.]*$/, ""),
           stats: {
             mtime: stats.mtime,
@@ -478,7 +450,11 @@ function scanDir(parent, path) {
           },
           size: stats.size,
           album: album
-        });
+        }
+        if (file != asset.filename) {
+          asset.raw = file;
+        }
+        assets.push(asset);
       });
     });
   }).then(function() {