jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Switched from ufraw to darktable as ufraw is no longer in Ubuntu

Browse Source 
Signed-off-by: James Ketrenos <james_gitlab@ketrenos.com>
This commit is contained in:
James Ketrenos 2020-01-03 16:40:21 -08:00
parent 6f51d5dc4d
commit 44b157ee8b
3 changed files with 98 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Dockerfile
View File

@ -13,8 +13,8 @@ RUN DEBIAN_FRONTEND=NONINTERACTIVE apt-get install -y \
make \
cmake
# You can then install the latest npm, polymer-cli, and bower:
RUN npm install --global npm@latest npx
# You can then install the latest npm and npx
RUN npm install --global npm@latest
# Speed up face-recognition and dev tools
RUN apt-get install -y libopenblas-dev
@ -22,13 +22,13 @@ RUN apt-get install -y libopenblas-dev
# Required for dlib to build
RUN apt-get install -y libx11-dev libpng16-16
# NEF processing uses ufraw-batch
RUN apt-get install -y ufraw-batch
# NEF processing uses darktable-cli, provided via darktable
RUN apt-get install -y darktable
# Create a user with sudo access
RUN DEBIAN_FRONTEND=noninteractive \
&& apt-get install --no-install-recommends -y \
sudo
sudo
# NOTE: Requires 'sudo' package to already be installed
RUN groupadd -g 1000 user \

8
README.md
View File

@ -24,12 +24,8 @@ wget -qO- https://deb.nodesource.com/setup_8.x | sudo bash -
sudo apt-get install --yes nodejs
```
You can then install the latest npm, polymer-cli, and bower:
```bash
sudo npm install --global npm@latest
sudo npm install --global polymer-cli
sudo npm install --global bower
```
# Install BLAS to improve performance, and dev tools so
@ -40,9 +36,9 @@ sudo apt install -y libopenblas-dev cmake
```
###
NEF processing uses ufraw-batch
NEF processing uses darktable
```
sudo apt install -y ufraw-batch
sudo apt install -y darktable
```
### Create `photos` user for DB

206
server/scanner.js
View File

@ -61,8 +61,8 @@ const { spawn } = require('child_process');
const sharp = require("sharp"), exif = require("exif-reader");
function convertRawToJpg(path, file) {
setStatus("Converting " + path + file);
function convertRawToJpg(path, raw, file) {
setStatus(`Converting ${path}${raw} to ${file}.`);
path = picturesPath + path;
@ -73,30 +73,23 @@ function convertRawToJpg(path, file) {
return;
}
const ufraw = spawn("ufraw-batch", [
"--silent",
"--wb=camera",
"--rotate=camera",
"--out-type=jpg",
"--compression=90",
"--exif",
"--overwrite",
"--output", path + file.replace(rawExtension, ".jpg"),
const darktable = spawn("darktable-cli", [
path + raw,
path + file
]);
const stderr = [];
ufraw.stderr.on('data', function(data) {
darktable.stderr.on('data', function(data) {
stderr.push(data);
});
return new Promise(function(ufraw, resolve, reject) {
ufraw.on('exit', function(stderr, code, signal) {
return new Promise((resolve, reject) => {
darktable.on('exit', (code, signal) => {
if (signal || code != 0) {
let error = "UFRAW for " + path + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n";
let error = "darktable for " + path + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n";
setStatus(error, "error");
return moveCorrupt(path, file).then(function() {
setStatus("ufraw failed", "warn");
setStatus("darktable failed", "warn");
return reject(error);
}).catch(function(error) {
setStatus("moveCorrupt failed", "warn");
@ -104,9 +97,9 @@ function convertRawToJpg(path, file) {
});
}
return mkdir(path + "raw").then(function() {
fs.rename(path + file, path + "raw/" + file, function(err) {
fs.rename(path + raw, path + "raw/" + raw, function(err) {
if (err) {
setStatus("Unable to move RAW file: " + path + file, "error");
setStatus("Unable to move RAW file: " + path + raw, "error");
return reject(err);
}
return resolve();
@ -115,8 +108,8 @@ function convertRawToJpg(path, file) {
setStatus("mkdir failed", "warn");
return reject(error);
});
}.bind(this, ufraw));
}.bind(this, stderr));
});
});
});
});
}
@ -164,76 +157,78 @@ function processBlock(items) {
let toProcess = processing.length, lastMessage = moment();
setStatus("Items to be processed: " + toProcess);
return Promise.mapSeries(processing, function(asset) {
return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
asset.hash = hash;
return asset;
}).then(function(asset) {
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
"WHERE hash=:hash OR photoId=:id", {
replacements: asset,
type: photoDB.sequelize.QueryTypes.SELECT
}).then(function(results) {
let query;
return Promise.mapSeries(processing, (asset) => {
if (!asset.raw) {
return;
}
const path = asset.album.path;
if (results.length == 0) {
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
} else if (results[0].hash != asset.hash) {
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
} else if (results[0].photoId != asset.id) {
setStatus("Duplicate asset: " +
"'" + asset.album.path + asset.filename + "' is a copy of " +
"'" + results[0].path + results[0].filename + "'");
if (asset.duplicate != results[0].photoId) {
asset.duplicate = results[0].photoId;
duplicates.push(asset);
}
return null;
}
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
// console.log("process needed because of " + query);
needsProcessing.push(asset);
if (!query) {
return asset;
}
return photoDB.sequelize.query(query, {
replacements: asset,
}).then(function() {
return asset;
});
});
}).then(function(asset) {
if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */
return;
return exists(picturesPath + path + asset.filename).then(function(exist) {
if (exist) {
return asset;
}
return mkdir(picturesPath + path + "raw").then(function() {
return convertRawToJpg(path, asset.raw, asset.filename);
}).then(function() {
console.log("Done converting...");
});
});
}).then(() => {
return Promise.mapSeries(processing, (asset) => {
return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
asset.hash = hash;
return asset;
}).then(function(asset) {
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
"WHERE hash=:hash OR photoId=:id", {
replacements: asset,
type: photoDB.sequelize.QueryTypes.SELECT
}).then(function(results) {
let query;
var path = asset.album.path,
file = asset.filename,
created = asset.stats.mtime,
albumId = asset.album.id;
let tmp = Promise.resolve(file);
/* If this is a Nikon RAW file, convert it to JPG and move to /raw dir */
if (rawExtension.exec(file)) {
tmp = exists(picturesPath + path + file.replace(rawExtension, ".jpg")).then(function(exist) {
if (exist) {
return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
if (results.length == 0) {
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
} else if (results[0].hash != asset.hash) {
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
} else if (results[0].photoId != asset.id) {
setStatus("Duplicate asset: " +
"'" + asset.album.path + asset.filename + "' is a copy of " +
"'" + results[0].path + results[0].filename + "'");
if (asset.duplicate != results[0].photoId) {
asset.duplicate = results[0].photoId;
duplicates.push(asset);
}
return null;
}
return mkdir(picturesPath + path + "raw").then(function() {
return convertRawToJpg(path, file);
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
// console.log("process needed because of " + query);
needsProcessing.push(asset);
if (!query) {
return asset;
}
return photoDB.sequelize.query(query, {
replacements: asset,
}).then(function() {
return file.replace(rawExtension, ".jpg"); /* We converted from NEF/ORF => JPG */
return asset;
});
});
}
}).then(function(asset) {
if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */
return;
}
return tmp.then(function(file) {
var path = asset.album.path,
file = asset.filename,
created = asset.stats.mtime,
albumId = asset.album.id;
var src = picturesPath + path + file,
image = sharp(src);
@ -321,38 +316,15 @@ function processBlock(items) {
});
}).catch(function(error) {
setStatus("Error reading image " + src + ":\n" + error, "error");
return moveCorrupt(path, file).then(function() {
/* If the original file was not a NEF/ORF, we are done... */
if (!rawExtension.exec(asset.filename)) {
return;
}
/* ... otherwise, attempt to re-convert the NEF/ORF->JPG and then resize again */
for (var i = 0; i < triedClean.length; i++) {
if (triedClean[i] == path + file) {
/* Move the NEF/ORF to /corrupt as well so it isn't re-checked again and again... */
// return moveCorrupt(path, asset.filename);
setStatus("Already attempted to convert NEF/ORF to JPG: " + path + file, "error");
return;
}
}
setStatus("Adding " + path + file + " back onto processing queue.", "error");
triedClean.push(path + file);
processBlock([ path, file, created, albumId ]);
});
return moveCorrupt(path, file);
});
}).catch(function() {
setStatus("Continuing file processing.", "warn");
}).then(function() {
toProcess--;
if (moment().add(-5, 'seconds') > lastMessage) {
setStatus("Items to be processed: " + toProcess);
lastMessage = moment();
}
});
}).then(function() {
toProcess--;
if (moment().add(-5, 'seconds') > lastMessage) {
setStatus("Items to be processed: " + toProcess);
lastMessage = moment();
}
});
}).catch(function(error) {
setStatus("Error processing file. Continuing.", "error");
@ -469,8 +441,8 @@ function scanDir(parent, path) {
album.hasAssets = true;
assets.push({
filename: file.replace(rawExtension, ".jpg"), /* We will be converting from NEF/ORF => JPG */
const asset = {
filename: file.replace(rawExtension, ".jpg"),
name: file.replace(/.[^.]*$/, ""),
stats: {
mtime: stats.mtime,
@ -478,7 +450,11 @@ function scanDir(parent, path) {
},
size: stats.size,
album: album
});
}
if (file != asset.filename) {
asset.raw = file;
}
assets.push(asset);
});
});
}).then(function() {