Restructured scanner to be a little more maintainable
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
parent
d9f85bed30
commit
7fe6ae43ab
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,3 +1,5 @@
|
||||
*.pyc
|
||||
.env
|
||||
node_modules
|
||||
bower_components
|
||||
frontend/auto-clusters.html
|
||||
|
||||
@ -17,4 +17,4 @@ services:
|
||||
- ${PWD}/db:/website/db
|
||||
- ${PWD}/config/local.json:/website/config/local.json
|
||||
- /opt/ketrface/models:/root/.deepface
|
||||
# - ${PWD}:/website
|
||||
- ${PWD}:/website
|
||||
|
||||
@ -5,7 +5,7 @@ const config = require("config"),
|
||||
Promise = require("bluebird"),
|
||||
picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
|
||||
|
||||
const stat = function (_path) {
|
||||
const stat = async (_path) => {
|
||||
if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) {
|
||||
_path = _path.substring(picturesPath.length);
|
||||
}
|
||||
@ -22,7 +22,7 @@ const stat = function (_path) {
|
||||
});
|
||||
}
|
||||
|
||||
const unlink = function (_path) {
|
||||
const unlink = async (_path) => {
|
||||
if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) {
|
||||
_path = _path.substring(picturesPath.length);
|
||||
}
|
||||
@ -39,7 +39,7 @@ const unlink = function (_path) {
|
||||
});
|
||||
}
|
||||
|
||||
const mkdir = function (_path) {
|
||||
const mkdir = async (_path) => {
|
||||
if (_path.indexOf(picturesPath) == 0) {
|
||||
_path = _path.substring(picturesPath.length);
|
||||
}
|
||||
@ -72,7 +72,7 @@ const mkdir = function (_path) {
|
||||
});
|
||||
}
|
||||
|
||||
const exists = function(path) {
|
||||
const exists = async (path) => {
|
||||
return stat(path).then(function() {
|
||||
return true;
|
||||
}).catch(function() {
|
||||
|
||||
@ -134,8 +134,131 @@ function moveCorrupt(path, file) {
|
||||
});
|
||||
}
|
||||
|
||||
function processBlock(items) {
|
||||
const determineImageDate = (asset, metadata) => {
|
||||
const created = asset.stats.mtime,
|
||||
filename = asset.filename;
|
||||
|
||||
/* Attempt to find CREATED / MODIFIED date based on meta-data or
|
||||
* FILENAME patterns */
|
||||
if (metadata.exif
|
||||
&& metadata.exif.exif
|
||||
&& metadata.exif.exif.DateTimeOriginal
|
||||
&& !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
|
||||
asset.taken = moment(metadata.exif.exif.DateTimeOriginal).format();
|
||||
asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format();
|
||||
|
||||
if (asset.taken == "Invalid date"
|
||||
|| asset.taken.replace(/T.*/, "") == "1899-11-30") {
|
||||
setStatus(
|
||||
`Invalid EXIF date information for ` +
|
||||
`${asset.album.path + asset.filename}`);
|
||||
return moment(created).format();
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/* Attempt to infer the datestamp from the filename */
|
||||
let date = moment(created).format();
|
||||
|
||||
let match = filename.match(
|
||||
/WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/);
|
||||
|
||||
if (match) {
|
||||
date = moment((match[1] + " " + match[2]), "YYYY-MM-DD h.mm.ss a")
|
||||
.format();
|
||||
if (date == "Invalid date") {
|
||||
date = moment(created).format();
|
||||
}
|
||||
return date;
|
||||
}
|
||||
|
||||
match = filename.match(
|
||||
/(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/);
|
||||
if (!match) {
|
||||
return moment(created).format();
|
||||
}
|
||||
|
||||
if (match[2]) { /* Stamp had time in it */
|
||||
date = moment(`${match[1]}${match[2]}`
|
||||
.replace(/-/g, ""), "YYYYMMDDHHmmss")
|
||||
.format();
|
||||
} else {
|
||||
date = moment(`${match[1]}`
|
||||
.replace(/-/g, ""), "YYYYMMDD")
|
||||
.format();
|
||||
}
|
||||
|
||||
if (date == "Invalid date") {
|
||||
date = moment(created).format();
|
||||
}
|
||||
|
||||
return date;
|
||||
}
|
||||
|
||||
const processImageAsset = async (asset) => {
|
||||
let path = asset.album.path,
|
||||
filename = asset.filename;
|
||||
|
||||
let src = picturesPath + path + filename,
|
||||
image = sharp(src);
|
||||
|
||||
const metadata = await image/*.limitInputPixels(1073741824)*/
|
||||
.metadata()
|
||||
.catch(error => console.error(error) );
|
||||
|
||||
if (metadata.exif) {
|
||||
try {
|
||||
metadata.exif = exif(metadata.exif);
|
||||
delete metadata.exif.thumbnail;
|
||||
delete metadata.exif.image;
|
||||
for (var key in metadata.exif.exif) {
|
||||
if (Buffer.isBuffer(metadata.exif.exif[key])) {
|
||||
metadata.exif.exif[key] =
|
||||
"Buffer[" + metadata.exif.exif[key].length + "]";
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
delete metadata.exif
|
||||
}
|
||||
}
|
||||
|
||||
asset.width = metadata.width;
|
||||
asset.height = metadata.height;
|
||||
asset.added = moment().format();
|
||||
|
||||
const updateDate = determineImageDate(asset, metadata);
|
||||
if (updateDate) {
|
||||
asset.taken = asset.modified = updateDate;
|
||||
}
|
||||
|
||||
let dst = picturesPath + path + "thumbs/" + filename;
|
||||
|
||||
let onDisk = await exists(dst);
|
||||
if (!onDisk) {
|
||||
await image.resize(256, 256)
|
||||
.withMetadata()
|
||||
.toFile(dst)
|
||||
.catch((error) => {
|
||||
setStatus(`Error resizing image: ${dst}\n${error}`, "error");
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
dst = picturesPath + path + "thumbs/scaled/" + filename;
|
||||
if (!onDisk) {
|
||||
await image.resize(Math.min(1024, metadata.width))
|
||||
.withMetadata()
|
||||
.toFile(dst)
|
||||
.catch((error) => {
|
||||
setStatus(`Error resizing image: ${dst}\n${error}`, "error");
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const processBlock = async (items) => {
|
||||
if (items) {
|
||||
processQueue = processQueue.concat(items);
|
||||
}
|
||||
@ -146,7 +269,9 @@ function processBlock(items) {
|
||||
return;
|
||||
}
|
||||
|
||||
let processing = processQueue.splice(0), needsProcessing = [], duplicates = [];
|
||||
let processing = processQueue.splice(0),
|
||||
needsProcessing = [],
|
||||
duplicates = [];
|
||||
|
||||
processRunning = true;
|
||||
|
||||
@ -156,239 +281,147 @@ function processBlock(items) {
|
||||
});
|
||||
|
||||
let toProcess = processing.length, lastMessage = moment();
|
||||
setStatus("Items to be processed: " + toProcess);
|
||||
return Promise.mapSeries(processing, (asset) => {
|
||||
if (!asset.raw) {
|
||||
|
||||
setStatus("Items to be processed: " + toProcess);
|
||||
|
||||
await Promise.mapSeries(processing, async (asset) => {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
setStatus("Items to be processed: " + toProcess);
|
||||
lastMessage = moment();
|
||||
}
|
||||
|
||||
/* Create JPG from RAW if there is a RAW file and no JPG */
|
||||
if (asset.raw) {
|
||||
const path = asset.album.path;
|
||||
const onDisk = await exists(picturesPath + path + asset.filename)
|
||||
if (!onDisk) {
|
||||
await mkdir(picturesPath + path + "raw");
|
||||
await convertRawToJpg(path, asset.raw, asset.filename);
|
||||
console.log("Done converting...");
|
||||
}
|
||||
}
|
||||
|
||||
/* Update PHOTOHASH table */
|
||||
asset.hash = await computeHash(
|
||||
picturesPath + asset.album.path + asset.filename)
|
||||
|
||||
let results = await photoDB.sequelize.query(
|
||||
"SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
|
||||
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
|
||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
||||
"WHERE hash=:hash OR photoId=:id", {
|
||||
replacements: asset,
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
});
|
||||
|
||||
let query;
|
||||
|
||||
if (results.length == 0) {
|
||||
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
|
||||
} else if (results[0].hash != asset.hash) {
|
||||
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
|
||||
} else if (results[0].photoId != asset.id) {
|
||||
setStatus("Duplicate asset: " +
|
||||
"'" + asset.album.path + asset.filename + "' is a copy of " +
|
||||
"'" + results[0].path + results[0].filename + "'");
|
||||
if (asset.duplicate != results[0].photoId) {
|
||||
asset.duplicate = results[0].photoId;
|
||||
duplicates.push(asset);
|
||||
}
|
||||
return; /* Done processing this asset (DUPLICATE) */
|
||||
}
|
||||
|
||||
/* Update PHOTOHASH table if necessary */
|
||||
if (query) {
|
||||
await photoDB.sequelize.query(query, {
|
||||
replacements: asset,
|
||||
});
|
||||
}
|
||||
|
||||
/* Additional processing is only skipped if the asset was a
|
||||
* DUPLICATE above (the empty "return;") */
|
||||
|
||||
needsProcessing.push(asset);
|
||||
|
||||
try {
|
||||
await processImageAsset(asset)
|
||||
} catch (error) {
|
||||
const path = asset.album.path,
|
||||
filename = asset.filename;
|
||||
setStatus(`Error reading image ` +
|
||||
`${picturesPath}${path}${filename}:\n` +
|
||||
`${error}`, "error");
|
||||
await moveCorrupt(path, filename);
|
||||
return;
|
||||
}
|
||||
|
||||
const path = asset.album.path;
|
||||
|
||||
return exists(picturesPath + path + asset.filename).then(function(exist) {
|
||||
if (exist) {
|
||||
return asset;
|
||||
}
|
||||
|
||||
return mkdir(picturesPath + path + "raw").then(function() {
|
||||
return convertRawToJpg(path, asset.raw, asset.filename);
|
||||
}).then(function() {
|
||||
console.log("Done converting...");
|
||||
});
|
||||
/* Update the DB with the image information */
|
||||
await photoDB.sequelize.query("UPDATE photos SET " +
|
||||
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " +
|
||||
"WHERE id=:id", {
|
||||
replacements: asset,
|
||||
});
|
||||
}).then(() => {
|
||||
return Promise.mapSeries(processing, (asset) => {
|
||||
return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
|
||||
asset.hash = hash;
|
||||
return asset;
|
||||
}).then(function(asset) {
|
||||
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
|
||||
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
|
||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
||||
"WHERE hash=:hash OR photoId=:id", {
|
||||
replacements: asset,
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}).then(function(results) {
|
||||
let query;
|
||||
|
||||
if (results.length == 0) {
|
||||
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
|
||||
} else if (results[0].hash != asset.hash) {
|
||||
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
|
||||
} else if (results[0].photoId != asset.id) {
|
||||
setStatus("Duplicate asset: " +
|
||||
"'" + asset.album.path + asset.filename + "' is a copy of " +
|
||||
"'" + results[0].path + results[0].filename + "'");
|
||||
if (asset.duplicate != results[0].photoId) {
|
||||
asset.duplicate = results[0].photoId;
|
||||
duplicates.push(asset);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
|
||||
// console.log("process needed because of " + query);
|
||||
needsProcessing.push(asset);
|
||||
|
||||
if (!query) {
|
||||
return asset;
|
||||
}
|
||||
|
||||
return photoDB.sequelize.query(query, {
|
||||
replacements: asset,
|
||||
}).then(function() {
|
||||
return asset;
|
||||
});
|
||||
});
|
||||
}).then(function(asset) {
|
||||
if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */
|
||||
return;
|
||||
}
|
||||
|
||||
var path = asset.album.path,
|
||||
file = asset.filename,
|
||||
created = asset.stats.mtime,
|
||||
albumId = asset.album.id;
|
||||
|
||||
var src = picturesPath + path + file,
|
||||
image = sharp(src);
|
||||
|
||||
return image/*.limitInputPixels(1073741824)*/
|
||||
.metadata()
|
||||
.catch(error => {
|
||||
console.error(error);
|
||||
})
|
||||
.then((metadata) => {
|
||||
if (metadata.exif) {
|
||||
try {
|
||||
metadata.exif = exif(metadata.exif);
|
||||
delete metadata.exif.thumbnail;
|
||||
delete metadata.exif.image;
|
||||
for (var key in metadata.exif.exif) {
|
||||
if (Buffer.isBuffer(metadata.exif.exif[key])) {
|
||||
metadata.exif.exif[key] =
|
||||
"Buffer[" + metadata.exif.exif[key].length + "]";
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
delete metadata.exif
|
||||
}
|
||||
}
|
||||
|
||||
asset.width = metadata.width;
|
||||
asset.height = metadata.height;
|
||||
asset.added = moment().format();
|
||||
|
||||
if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
|
||||
asset.taken = moment(metadata.exif.exif.DateTimeOriginal).format();
|
||||
asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format();
|
||||
|
||||
if (asset.taken == "Invalid date" || asset.taken.replace(/T.*/, "") == "1899-11-30") {
|
||||
setStatus("Invalid EXIF date information for " + asset.album.path + asset.filename);
|
||||
asset.taken = asset.modified = moment(created).format();
|
||||
}
|
||||
} else {
|
||||
/* Attempt to infer the datestamp from the filename */
|
||||
let date = moment(created).format();
|
||||
|
||||
let match = file.match(/WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/);
|
||||
if (match) {
|
||||
date = moment((match[1]+" "+match[2]), "YYYY-MM-DD h.mm.ss a").format();
|
||||
if (date == "Invalid date") {
|
||||
date = moment(created).format();
|
||||
}
|
||||
} else {
|
||||
match = file.match(/(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/);
|
||||
if (match) {
|
||||
if (match[2]) { /* Stamp had time in it */
|
||||
date = moment((match[1]+""+match[2]).replace(/-/g, ""), "YYYYMMDDHHmmss").format();
|
||||
} else {
|
||||
date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
|
||||
}
|
||||
if (date == "Invalid date") {
|
||||
date = moment(created).format();
|
||||
}
|
||||
} else {
|
||||
date = moment(created).format();
|
||||
}
|
||||
}
|
||||
asset.taken = asset.modified = date;
|
||||
}
|
||||
|
||||
let dst = picturesPath + path + "thumbs/" + file;
|
||||
|
||||
return exists(dst).then(function(exist) {
|
||||
if (exist) {
|
||||
return;
|
||||
}
|
||||
|
||||
return image.resize(256, 256).withMetadata().toFile(dst).catch(function(error) {
|
||||
setStatus("Error resizing image: " + dst + "\n" + error, "error");
|
||||
throw error;
|
||||
});
|
||||
}).then(function() {
|
||||
let dst = picturesPath + path + "thumbs/scaled/" + file;
|
||||
return exists(dst).then(function(exist) {
|
||||
if (exist) {
|
||||
return;
|
||||
}
|
||||
|
||||
return image.resize(Math.min(1024, metadata.width)).withMetadata().toFile(dst).catch(function(error) {
|
||||
setStatus("Error resizing image: " + dst + "\n" + error, "error");
|
||||
throw error;
|
||||
});
|
||||
});
|
||||
}).then(function() {
|
||||
return photoDB.sequelize.query("UPDATE photos SET " +
|
||||
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " +
|
||||
"WHERE id=:id", {
|
||||
replacements: asset,
|
||||
});
|
||||
});
|
||||
}).catch(function(error) {
|
||||
setStatus("Error reading image " + src + ":\n" + error, "error");
|
||||
return moveCorrupt(path, file);
|
||||
});
|
||||
}).then(function() {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
setStatus("Items to be processed: " + toProcess);
|
||||
lastMessage = moment();
|
||||
}
|
||||
});
|
||||
});
|
||||
}).catch(function(error) {
|
||||
setStatus("Error processing file. Continuing.", "error");
|
||||
throw error;
|
||||
}).then(function() {
|
||||
setStatus("Completed processing queue. Marking " + duplicates.length + " duplicates.");
|
||||
return photoDB.sequelize.transaction(function(transaction) {
|
||||
return Promise.mapSeries(duplicates, function(asset) {
|
||||
return photoDB.sequelize.query("UPDATE photos " +
|
||||
"SET duplicate=:duplicate,modified=CURRENT_TIMESTAMP,scanned=CURRENT_TIMESTAMP WHERE id=:id", {
|
||||
replacements: asset,
|
||||
transaction: transaction
|
||||
});
|
||||
});
|
||||
});
|
||||
}).then(function() {
|
||||
setStatus("Looking for removed assets");
|
||||
return photoDB.sequelize.query("SELECT photos.scanned,photos.id,photos.filename,albums.path FROM photos " +
|
||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
||||
"WHERE photos.deleted=0 AND (DATETIME(photos.scanned)<DATETIME(:lastScan) OR photos.scanned IS NULL)", {
|
||||
replacements: {
|
||||
lastScan: lastScan
|
||||
},
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}).then(function(results) {
|
||||
let deleted = [];
|
||||
setStatus("Checking " + results.length + " assets to see if they are on disk.");
|
||||
return Promise.map(results, function(asset) {
|
||||
return exists(asset.path + asset.filename).then(function(exist) {
|
||||
if (!exist) {
|
||||
setStatus(asset.path + asset.filename + " no longer exists on disk. Marking as deleted.");
|
||||
deleted.push(asset.id);
|
||||
}
|
||||
});
|
||||
}).then(function() {
|
||||
return photoDB.sequelize.query("UPDATE photos SET deleted=1,scanned=CURRENT_TIMESTAMP WHERE id IN (:deleted)", {
|
||||
replacements: {
|
||||
deleted: deleted
|
||||
}
|
||||
}).then(function() {
|
||||
return photoDB.sequelize.query("DELETE FROM photohashes WHERE photoId IN (:deleted)", {
|
||||
replacements: {
|
||||
deleted: deleted
|
||||
}
|
||||
});
|
||||
}).then(function() {
|
||||
setStatus(deleted.length + " assets deleted.");
|
||||
});
|
||||
});
|
||||
});
|
||||
}).then(function() {
|
||||
processRunning = false;
|
||||
});
|
||||
|
||||
/* Process the DUPLICATES */
|
||||
setStatus(
|
||||
`Completed processing queue. Marking ${duplicates.length} duplicates.`);
|
||||
await photoDB.sequelize.transaction(async (t) => {
|
||||
await Promise.mapSeries(duplicates, async (item) => {
|
||||
await photoDB.sequelize.query(
|
||||
"UPDATE photos " +
|
||||
"SET duplicate=:duplicate,modified=CURRENT_TIMESTAMP,scanned=CURRENT_TIMESTAMP WHERE id=:id", {
|
||||
replacements: item,
|
||||
transaction: t
|
||||
});
|
||||
});
|
||||
})
|
||||
|
||||
setStatus("Looking for removed assets");
|
||||
let results = await photoDB.sequelize.query(
|
||||
"SELECT photos.scanned,photos.id,photos.filename,albums.path FROM photos " +
|
||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
||||
"WHERE photos.deleted=0 AND (DATETIME(photos.scanned)<DATETIME(:lastScan) OR photos.scanned IS NULL)", {
|
||||
replacements: {
|
||||
lastScan: lastScan
|
||||
},
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
});
|
||||
|
||||
let deleted = [];
|
||||
setStatus(`Checking ${results.length} assets to see if they are on disk.`);
|
||||
await Promise.map(results, async (item) => {
|
||||
const onDisk = await exists(item.path + item.filename);
|
||||
if (!onDisk) {
|
||||
setStatus(
|
||||
`${item.path}${item.filename} no longer exists on disk. ` +
|
||||
`Marking as deleted.`);
|
||||
deleted.push(item.id);
|
||||
}
|
||||
});
|
||||
|
||||
if (deleted.length) {
|
||||
await photoDB.sequelize.transaction(async (t) => {
|
||||
await photoDB.sequelize.query(
|
||||
"UPDATE photos SET deleted=1,scanned=CURRENT_TIMESTAMP " +
|
||||
"WHERE id IN (:deleted)", {
|
||||
replacements: { deleted },
|
||||
transaction: t
|
||||
}
|
||||
);
|
||||
await photoDB.sequelize.query(
|
||||
"DELETE FROM photohashes WHERE photoId IN (:deleted)", {
|
||||
replacements: { deleted },
|
||||
transaction: t
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
setStatus(`${deleted.length} assets deleted.`);
|
||||
|
||||
processRunning = false;
|
||||
}
|
||||
|
||||
function scanDir(parent, path) {
|
||||
@ -435,7 +468,9 @@ function scanDir(parent, path) {
|
||||
return stat(filepath).then(function(stats) {
|
||||
if (stats.isDirectory()) {
|
||||
filepath += "/";
|
||||
return scanDir(album, filepath).spread(function(_albums, _assets) {
|
||||
return scanDir(album, filepath).then((res) => {
|
||||
const _albums = res.albums,
|
||||
_assets = res.assets;
|
||||
album.allAssetCount += _assets.length;
|
||||
album.allAlbumCount += _albums.length + 1;
|
||||
albums = albums.concat(_albums);
|
||||
@ -475,11 +510,11 @@ function scanDir(parent, path) {
|
||||
}
|
||||
});
|
||||
}).then(function() {
|
||||
return [ albums, assets ];
|
||||
return { albums, assets };
|
||||
});
|
||||
}
|
||||
|
||||
function findOrCreateDBAlbum(transaction, album) {
|
||||
const findOrCreateDBAlbum = async (t, album) => {
|
||||
let query = "SELECT id FROM albums WHERE path=:path AND ";
|
||||
if (!album.parent) {
|
||||
query += "parentId IS NULL";
|
||||
@ -494,30 +529,29 @@ function findOrCreateDBAlbum(transaction, album) {
|
||||
query += "parentId=:parentId";
|
||||
}
|
||||
|
||||
return photoDB.sequelize.query(query, {
|
||||
const results = await photoDB.sequelize.query(query, {
|
||||
replacements: album,
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}).then(function(results) {
|
||||
if (results.length == 0) {
|
||||
if (!album.parent) {
|
||||
setStatus("Creating top level album: " + picturesPath, "warn" );
|
||||
}
|
||||
return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
|
||||
replacements: album,
|
||||
transaction: transaction
|
||||
}).then(array => {
|
||||
return array[1].lastID;
|
||||
});
|
||||
} else {
|
||||
return results[0].id;
|
||||
}
|
||||
}).then(function(id) {
|
||||
album.id = id;
|
||||
return id;
|
||||
});
|
||||
|
||||
if (results.length == 0) {
|
||||
if (!album.parent) {
|
||||
setStatus("Creating top level album: " + picturesPath, "warn" );
|
||||
}
|
||||
return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
|
||||
replacements: album,
|
||||
transaction: t
|
||||
}).then(array => {
|
||||
album.id = array[1].lastID;
|
||||
});
|
||||
} else {
|
||||
album.id = results[0].id;
|
||||
}
|
||||
|
||||
return album.id;
|
||||
}
|
||||
|
||||
const findOrUpdateDBAsset = async (transaction, asset) => {
|
||||
const findOrUpdateDBAsset = async (t, asset) => {
|
||||
if (!asset.album || !asset.album.id) {
|
||||
let error = "Asset being processed without an album";
|
||||
setStatus(error, "warn");
|
||||
@ -538,7 +572,7 @@ const findOrUpdateDBAsset = async (transaction, asset) => {
|
||||
return await photoDB.sequelize.query("INSERT INTO photos " +
|
||||
"(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", {
|
||||
replacements: asset,
|
||||
transaction: transaction
|
||||
transaction: t
|
||||
}).then(array => {
|
||||
asset.id = array[1].lastID;
|
||||
return asset;
|
||||
@ -615,7 +649,7 @@ function setStatus(status, level) {
|
||||
}
|
||||
}
|
||||
|
||||
function doScan() {
|
||||
const doScan = async () => {
|
||||
/* 1. Scan for all assets which will be managed by the system. readdir
|
||||
* 2. Check if entry in DB. Check mod-time in DB vs. stats from #1
|
||||
* - For albums
|
||||
@ -636,119 +670,156 @@ function doScan() {
|
||||
let needsProcessing = [];
|
||||
|
||||
if (scanningStatus.length != 0) {
|
||||
return Promise.resolve(scanningStatus);
|
||||
return scanningStatus;
|
||||
}
|
||||
|
||||
return scanDir(null, picturesPath).spread(function(albums, assets) {
|
||||
setStatus("Found " + assets.length + " assets in " + albums.length + " albums after " +
|
||||
((Date.now() - now) / 1000) + "s");
|
||||
/* One at a time, in series, as the album[] array has parents first, then descendants.
|
||||
* Operating in parallel could result in a child being searched for prior to the parent */
|
||||
now = Date.now();
|
||||
let { albums, assets } = await scanDir(null, picturesPath);
|
||||
|
||||
let toProcess = albums.length, lastMessage = moment();
|
||||
return photoDB.sequelize.transaction(function(transaction) {
|
||||
return Promise.mapSeries(albums, function(album) {
|
||||
return findOrCreateDBAlbum(transaction, album).then(function() {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
setStatus("Albums to be created in DB: " + toProcess);
|
||||
lastMessage = moment();
|
||||
}
|
||||
});
|
||||
});
|
||||
}).then(function() {
|
||||
setStatus("Processed " + albums.length + " album DB entries in " +
|
||||
((Date.now() - now) / 1000) + "s");
|
||||
now = Date.now();
|
||||
setStatus(
|
||||
`Found ${assets.length} assets in ${albums.length} albums after ` +
|
||||
`${(Date.now() - now) / 1000}s`);
|
||||
|
||||
/* One at a time, in series, as the album[] array has parents
|
||||
* first, then descendants.
|
||||
* Operating in parallel could result in a child being searched for
|
||||
* prior to the parent */
|
||||
now = Date.now();
|
||||
|
||||
setStatus(assets.length + " assets remaining to be verified/updated. ETA N/A");
|
||||
|
||||
let processed = 0, start = Date.now(), last = 0, updateScanned = [], newEntries = 0;
|
||||
return photoDB.sequelize.transaction(function(transaction) {
|
||||
return Promise.map(assets, function(asset) {
|
||||
return Promise.resolve(asset).then(function(asset) {
|
||||
/* If both mtime and ctime of the asset are older than the lastScan, skip it
|
||||
*
|
||||
* Can only do this after a full scan has occurred */
|
||||
if (lastScan != null && asset.stats.mtime < lastScan && asset.stats.ctime < lastScan) {
|
||||
return asset;
|
||||
}
|
||||
|
||||
return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
|
||||
if (!asset.scanned) {
|
||||
newEntries++;
|
||||
}
|
||||
if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) {
|
||||
// if (!asset.scanned) { console.log("no scan date on asset"); }
|
||||
// if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); }
|
||||
// if (!asset.modified) { console.log("no mtime."); }
|
||||
needsProcessing.push(asset);
|
||||
} else {
|
||||
updateScanned.push(asset.id);
|
||||
}
|
||||
return asset;
|
||||
}).then(function(asset) {
|
||||
return asset;
|
||||
});
|
||||
}).then(function(asset) {
|
||||
processed++;
|
||||
|
||||
let elapsed = Date.now() - start;
|
||||
if (elapsed < 5000) {
|
||||
return asset;
|
||||
}
|
||||
|
||||
let remaining = assets.length - processed, eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
|
||||
setStatus(remaining + " assets remaining be verified/updated " +
|
||||
"(" + newEntries + " new entries, " + needsProcessing.length + " need processing," + (processed - newEntries) + " up-to-date so far). ETA " + eta + "s");
|
||||
last = processed;
|
||||
start = Date.now();
|
||||
});
|
||||
}, {
|
||||
concurrency: 10
|
||||
});
|
||||
}).then(function() {
|
||||
if (updateScanned.length) {
|
||||
return photoDB.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:ids)", {
|
||||
replacements: {
|
||||
ids: updateScanned
|
||||
}
|
||||
}).then(function() {
|
||||
setStatus("Updated scan date of " + updateScanned.length + " assets");
|
||||
updateScanned = [];
|
||||
});
|
||||
try {
|
||||
let toProcess = albums.length,
|
||||
lastMessage = moment();
|
||||
await photoDB.sequelize.transaction(async (t) => {
|
||||
await Promise.mapSeries(albums, async (album) => {
|
||||
await findOrCreateDBAlbum(t, album);
|
||||
|
||||
toProcess--;
|
||||
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
setStatus(`Albums to be created in DB: ${toProcess}`);
|
||||
lastMessage = moment();
|
||||
}
|
||||
}).then(function() {
|
||||
setStatus(newEntries + " assets are new. " + (needsProcessing.length - newEntries) + " assets have been modified.\n" +
|
||||
needsProcessing.length + " assets need HASH computed. " + (assets.length - needsProcessing.length) + " need no update.");;
|
||||
processBlock(needsProcessing);
|
||||
needsProcessing = [];
|
||||
}).then(function() {
|
||||
setStatus("Scanned " + assets.length + " asset DB entries in " +
|
||||
((Date.now() - now) / 1000) + "s");
|
||||
assets = [];
|
||||
});
|
||||
});
|
||||
}).then(function() {
|
||||
setStatus("Total time to initialize DB and all scans: " + ((Date.now() - initialized) / 1000) + "s");
|
||||
return photoDB.sequelize.query("SELECT max(scanned) AS scanned FROM photos", {
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
}).then(function(results) {
|
||||
if (results[0].scanned == null) {
|
||||
lastScan = new Date("1800-01-01");
|
||||
} else {
|
||||
lastScan = new Date(results[0].scanned);
|
||||
}
|
||||
setStatus("Updating any asset newer than " + moment(lastScan).format());
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
setStatus(
|
||||
`Processed ${albums.length} album DB entries in ` +
|
||||
`${(Date.now() - now) / 1000}s`);
|
||||
|
||||
now = Date.now();
|
||||
|
||||
setStatus(
|
||||
`${assets.length} assets remaining to be verified/updated. ETA N/A`);
|
||||
|
||||
let updateScanned = [],
|
||||
newEntries = 0;
|
||||
|
||||
try {
|
||||
let processed = 0,
|
||||
start = Date.now(),
|
||||
last = 0;
|
||||
|
||||
await photoDB.sequelize.transaction(async (t) => {
|
||||
await Promise.map(assets, async (asset) => {
|
||||
/* If both mtime and ctime of the asset are older than the
|
||||
* lastScan, skip it
|
||||
* Can only do this after a full scan has occurred */
|
||||
if (lastScan != null
|
||||
&& asset.stats.mtime < lastScan
|
||||
&& asset.stats.ctime < lastScan) {
|
||||
return;
|
||||
}
|
||||
|
||||
asset = await findOrUpdateDBAsset(t, asset);
|
||||
if (!asset.scanned) {
|
||||
newEntries++;
|
||||
}
|
||||
|
||||
if (!asset.scanned
|
||||
|| asset.scanned < asset.stats.mtime
|
||||
|| !asset.modified) {
|
||||
// if (!asset.scanned) { console.log("no scan date on asset"); }
|
||||
// if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); }
|
||||
// if (!asset.modified) { console.log("no mtime."); }
|
||||
needsProcessing.push(asset);
|
||||
} else {
|
||||
updateScanned.push(asset.id);
|
||||
}
|
||||
|
||||
processed++;
|
||||
|
||||
let elapsed = Date.now() - start;
|
||||
if (elapsed < 5000) {
|
||||
return;
|
||||
}
|
||||
|
||||
let remaining = assets.length - processed,
|
||||
eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
|
||||
setStatus(
|
||||
`${remaining} assets remaining be verified/updated (${newEntries} ` +
|
||||
`new entries, ${needsProcessing.length} need processing, ` +
|
||||
`${(processed - newEntries)} up-to-date so far). ETA ${eta}s`
|
||||
);
|
||||
last = processed;
|
||||
start = Date.now();
|
||||
}, {
|
||||
concurrency: 10
|
||||
});
|
||||
});
|
||||
}).then(function() {
|
||||
setStatus("idle");
|
||||
return "scan complete";
|
||||
}).catch(function(error) {
|
||||
setStatus(error);
|
||||
throw error;
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
if (updateScanned.length) {
|
||||
await photoDB.sequelize.query(
|
||||
"UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:ids)", {
|
||||
replacements: {
|
||||
ids: updateScanned
|
||||
}
|
||||
});
|
||||
setStatus("Updated scan date of " + updateScanned.length + " assets");
|
||||
updateScanned = [];
|
||||
}
|
||||
|
||||
setStatus(
|
||||
`${newEntries} assets are new. ` +
|
||||
`${needsProcessing.length - newEntries} assets have been ` +
|
||||
`modified.\n${needsProcessing.length} assets need HASH computed. ` +
|
||||
`${assets.length - needsProcessing.length} need no update.`);
|
||||
|
||||
processBlock(needsProcessing);
|
||||
|
||||
needsProcessing = [];
|
||||
|
||||
setStatus(
|
||||
`Scanned ${assets.length} asset DB entries in ` +
|
||||
`${(Date.now() - now) / 1000}s`);
|
||||
assets = [];
|
||||
|
||||
setStatus(
|
||||
`Total time to initialize DB and all scans: ` +
|
||||
`${(Date.now() - initialized) / 1000}s`);
|
||||
|
||||
let results = await photoDB.sequelize.query(
|
||||
"SELECT max(scanned) AS scanned FROM photos", {
|
||||
type: photoDB.sequelize.QueryTypes.SELECT
|
||||
});
|
||||
|
||||
if (results[0].scanned == null) {
|
||||
lastScan = new Date("1800-01-01");
|
||||
} else {
|
||||
lastScan = new Date(results[0].scanned);
|
||||
}
|
||||
|
||||
setStatus(`Updating any asset newer than ${moment(lastScan).format()}`);
|
||||
|
||||
setStatus("idle");
|
||||
|
||||
return "scan complete";
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user