diff --git a/.gitignore b/.gitignore index 81ac384..071f4a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.pyc .env node_modules +bower_components frontend/auto-clusters.html diff --git a/docker-compose.yml b/docker-compose.yml index df12cd2..677b2ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,4 +17,4 @@ services: - ${PWD}/db:/website/db - ${PWD}/config/local.json:/website/config/local.json - /opt/ketrface/models:/root/.deepface -# - ${PWD}:/website + - ${PWD}:/website diff --git a/server/lib/util.js b/server/lib/util.js index dcbb545..bbddb82 100644 --- a/server/lib/util.js +++ b/server/lib/util.js @@ -5,7 +5,7 @@ const config = require("config"), Promise = require("bluebird"), picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/"; -const stat = function (_path) { +const stat = async (_path) => { if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) { _path = _path.substring(picturesPath.length); } @@ -22,7 +22,7 @@ const stat = function (_path) { }); } -const unlink = function (_path) { +const unlink = async (_path) => { if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) { _path = _path.substring(picturesPath.length); } @@ -39,7 +39,7 @@ const unlink = function (_path) { }); } -const mkdir = function (_path) { +const mkdir = async (_path) => { if (_path.indexOf(picturesPath) == 0) { _path = _path.substring(picturesPath.length); } @@ -72,7 +72,7 @@ const mkdir = function (_path) { }); } -const exists = function(path) { +const exists = async (path) => { return stat(path).then(function() { return true; }).catch(function() { diff --git a/server/scanner.js b/server/scanner.js index 867d7ed..f607d5c 100755 --- a/server/scanner.js +++ b/server/scanner.js @@ -134,8 +134,131 @@ function moveCorrupt(path, file) { }); } -function processBlock(items) { +const determineImageDate = (asset, metadata) => { + const created = asset.stats.mtime, + filename = asset.filename; + + /* Attempt to find CREATED / MODIFIED date based on meta-data or + * FILENAME patterns */ + if (metadata.exif + && metadata.exif.exif + && metadata.exif.exif.DateTimeOriginal + && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) { + asset.taken = moment(metadata.exif.exif.DateTimeOriginal).format(); + asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format(); + + if (asset.taken == "Invalid date" + || asset.taken.replace(/T.*/, "") == "1899-11-30") { + setStatus( + `Invalid EXIF date information for ` + + `${asset.album.path + asset.filename}`); + return moment(created).format(); + } + + return undefined; + } + + /* Attempt to infer the datestamp from the filename */ + let date = moment(created).format(); + + let match = filename.match( + /WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/); + if (match) { + date = moment((match[1] + " " + match[2]), "YYYY-MM-DD h.mm.ss a") + .format(); + if (date == "Invalid date") { + date = moment(created).format(); + } + return date; + } + + match = filename.match( + /(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/); + if (!match) { + return moment(created).format(); + } + + if (match[2]) { /* Stamp had time in it */ + date = moment(`${match[1]}${match[2]}` + .replace(/-/g, ""), "YYYYMMDDHHmmss") + .format(); + } else { + date = moment(`${match[1]}` + .replace(/-/g, ""), "YYYYMMDD") + .format(); + } + + if (date == "Invalid date") { + date = moment(created).format(); + } + + return date; +} + +const processImageAsset = async (asset) => { + let path = asset.album.path, + filename = asset.filename; + + let src = picturesPath + path + filename, + image = sharp(src); + + const metadata = await image/*.limitInputPixels(1073741824)*/ + .metadata() + .catch(error => console.error(error) ); + + if (metadata.exif) { + try { + metadata.exif = exif(metadata.exif); + delete metadata.exif.thumbnail; + delete metadata.exif.image; + for (var key in metadata.exif.exif) { + if (Buffer.isBuffer(metadata.exif.exif[key])) { + metadata.exif.exif[key] = + "Buffer[" + metadata.exif.exif[key].length + "]"; + } + } + } catch (error) { + console.error(error); + delete metadata.exif + } + } + + asset.width = metadata.width; + asset.height = metadata.height; + asset.added = moment().format(); + + const updateDate = determineImageDate(asset, metadata); + if (updateDate) { + asset.taken = asset.modified = updateDate; + } + + let dst = picturesPath + path + "thumbs/" + filename; + + let onDisk = await exists(dst); + if (!onDisk) { + await image.resize(256, 256) + .withMetadata() + .toFile(dst) + .catch((error) => { + setStatus(`Error resizing image: ${dst}\n${error}`, "error"); + throw error; + }); + } + + dst = picturesPath + path + "thumbs/scaled/" + filename; + if (!onDisk) { + await image.resize(Math.min(1024, metadata.width)) + .withMetadata() + .toFile(dst) + .catch((error) => { + setStatus(`Error resizing image: ${dst}\n${error}`, "error"); + throw error; + }); + } +}; + +const processBlock = async (items) => { if (items) { processQueue = processQueue.concat(items); } @@ -146,7 +269,9 @@ function processBlock(items) { return; } - let processing = processQueue.splice(0), needsProcessing = [], duplicates = []; + let processing = processQueue.splice(0), + needsProcessing = [], + duplicates = []; processRunning = true; @@ -156,239 +281,147 @@ function processBlock(items) { }); let toProcess = processing.length, lastMessage = moment(); - setStatus("Items to be processed: " + toProcess); - return Promise.mapSeries(processing, (asset) => { - if (!asset.raw) { + + setStatus("Items to be processed: " + toProcess); + + await Promise.mapSeries(processing, async (asset) => { + toProcess--; + if (moment().add(-5, 'seconds') > lastMessage) { + setStatus("Items to be processed: " + toProcess); + lastMessage = moment(); + } + + /* Create JPG from RAW if there is a RAW file and no JPG */ + if (asset.raw) { + const path = asset.album.path; + const onDisk = await exists(picturesPath + path + asset.filename) + if (!onDisk) { + await mkdir(picturesPath + path + "raw"); + await convertRawToJpg(path, asset.raw, asset.filename); + console.log("Done converting..."); + } + } + + /* Update PHOTOHASH table */ + asset.hash = await computeHash( + picturesPath + asset.album.path + asset.filename) + + let results = await photoDB.sequelize.query( + "SELECT photohashes.*,photos.filename,albums.path FROM photohashes " + + "LEFT JOIN photos ON (photos.id=photohashes.photoId) " + + "LEFT JOIN albums ON (albums.id=photos.albumId) " + + "WHERE hash=:hash OR photoId=:id", { + replacements: asset, + type: photoDB.sequelize.QueryTypes.SELECT + }); + + let query; + + if (results.length == 0) { + query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; + } else if (results[0].hash != asset.hash) { + query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id"; + } else if (results[0].photoId != asset.id) { + setStatus("Duplicate asset: " + + "'" + asset.album.path + asset.filename + "' is a copy of " + + "'" + results[0].path + results[0].filename + "'"); + if (asset.duplicate != results[0].photoId) { + asset.duplicate = results[0].photoId; + duplicates.push(asset); + } + return; /* Done processing this asset (DUPLICATE) */ + } + + /* Update PHOTOHASH table if necessary */ + if (query) { + await photoDB.sequelize.query(query, { + replacements: asset, + }); + } + + /* Additional processing is only skipped if the asset was a + * DUPLICATE above (the empty "return;") */ + + needsProcessing.push(asset); + + try { + await processImageAsset(asset) + } catch (error) { + const path = asset.album.path, + filename = asset.filename; + setStatus(`Error reading image ` + + `${picturesPath}${path}${filename}:\n` + + `${error}`, "error"); + await moveCorrupt(path, filename); return; } - - const path = asset.album.path; - return exists(picturesPath + path + asset.filename).then(function(exist) { - if (exist) { - return asset; - } - - return mkdir(picturesPath + path + "raw").then(function() { - return convertRawToJpg(path, asset.raw, asset.filename); - }).then(function() { - console.log("Done converting..."); - }); + /* Update the DB with the image information */ + await photoDB.sequelize.query("UPDATE photos SET " + + "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " + + "WHERE id=:id", { + replacements: asset, }); - }).then(() => { - return Promise.mapSeries(processing, (asset) => { - return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) { - asset.hash = hash; - return asset; - }).then(function(asset) { - return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " + - "LEFT JOIN photos ON (photos.id=photohashes.photoId) " + - "LEFT JOIN albums ON (albums.id=photos.albumId) " + - "WHERE hash=:hash OR photoId=:id", { - replacements: asset, - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(results) { - let query; - - if (results.length == 0) { - query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; - } else if (results[0].hash != asset.hash) { - query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id"; - } else if (results[0].photoId != asset.id) { - setStatus("Duplicate asset: " + - "'" + asset.album.path + asset.filename + "' is a copy of " + - "'" + results[0].path + results[0].filename + "'"); - if (asset.duplicate != results[0].photoId) { - asset.duplicate = results[0].photoId; - duplicates.push(asset); - } - return null; - } - - /* Even if the hash doesn't need to be updated, the entry needs to be scanned */ - // console.log("process needed because of " + query); - needsProcessing.push(asset); - - if (!query) { - return asset; - } - - return photoDB.sequelize.query(query, { - replacements: asset, - }).then(function() { - return asset; - }); - }); - }).then(function(asset) { - if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */ - return; - } - - var path = asset.album.path, - file = asset.filename, - created = asset.stats.mtime, - albumId = asset.album.id; - - var src = picturesPath + path + file, - image = sharp(src); - - return image/*.limitInputPixels(1073741824)*/ - .metadata() - .catch(error => { - console.error(error); - }) - .then((metadata) => { - if (metadata.exif) { - try { - metadata.exif = exif(metadata.exif); - delete metadata.exif.thumbnail; - delete metadata.exif.image; - for (var key in metadata.exif.exif) { - if (Buffer.isBuffer(metadata.exif.exif[key])) { - metadata.exif.exif[key] = - "Buffer[" + metadata.exif.exif[key].length + "]"; - } - } - } catch (error) { - console.error(error); - delete metadata.exif - } - } - - asset.width = metadata.width; - asset.height = metadata.height; - asset.added = moment().format(); - - if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) { - asset.taken = moment(metadata.exif.exif.DateTimeOriginal).format(); - asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format(); - - if (asset.taken == "Invalid date" || asset.taken.replace(/T.*/, "") == "1899-11-30") { - setStatus("Invalid EXIF date information for " + asset.album.path + asset.filename); - asset.taken = asset.modified = moment(created).format(); - } - } else { - /* Attempt to infer the datestamp from the filename */ - let date = moment(created).format(); - - let match = file.match(/WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/); - if (match) { - date = moment((match[1]+" "+match[2]), "YYYY-MM-DD h.mm.ss a").format(); - if (date == "Invalid date") { - date = moment(created).format(); - } - } else { - match = file.match(/(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/); - if (match) { - if (match[2]) { /* Stamp had time in it */ - date = moment((match[1]+""+match[2]).replace(/-/g, ""), "YYYYMMDDHHmmss").format(); - } else { - date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format(); - } - if (date == "Invalid date") { - date = moment(created).format(); - } - } else { - date = moment(created).format(); - } - } - asset.taken = asset.modified = date; - } - - let dst = picturesPath + path + "thumbs/" + file; - - return exists(dst).then(function(exist) { - if (exist) { - return; - } - - return image.resize(256, 256).withMetadata().toFile(dst).catch(function(error) { - setStatus("Error resizing image: " + dst + "\n" + error, "error"); - throw error; - }); - }).then(function() { - let dst = picturesPath + path + "thumbs/scaled/" + file; - return exists(dst).then(function(exist) { - if (exist) { - return; - } - - return image.resize(Math.min(1024, metadata.width)).withMetadata().toFile(dst).catch(function(error) { - setStatus("Error resizing image: " + dst + "\n" + error, "error"); - throw error; - }); - }); - }).then(function() { - return photoDB.sequelize.query("UPDATE photos SET " + - "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " + - "WHERE id=:id", { - replacements: asset, - }); - }); - }).catch(function(error) { - setStatus("Error reading image " + src + ":\n" + error, "error"); - return moveCorrupt(path, file); - }); - }).then(function() { - toProcess--; - if (moment().add(-5, 'seconds') > lastMessage) { - setStatus("Items to be processed: " + toProcess); - lastMessage = moment(); - } - }); - }); - }).catch(function(error) { - setStatus("Error processing file. Continuing.", "error"); - throw error; - }).then(function() { - setStatus("Completed processing queue. Marking " + duplicates.length + " duplicates."); - return photoDB.sequelize.transaction(function(transaction) { - return Promise.mapSeries(duplicates, function(asset) { - return photoDB.sequelize.query("UPDATE photos " + - "SET duplicate=:duplicate,modified=CURRENT_TIMESTAMP,scanned=CURRENT_TIMESTAMP WHERE id=:id", { - replacements: asset, - transaction: transaction - }); - }); - }); - }).then(function() { - setStatus("Looking for removed assets"); - return photoDB.sequelize.query("SELECT photos.scanned,photos.id,photos.filename,albums.path FROM photos " + - "LEFT JOIN albums ON (albums.id=photos.albumId) " + - "WHERE photos.deleted=0 AND (DATETIME(photos.scanned) { + await Promise.mapSeries(duplicates, async (item) => { + await photoDB.sequelize.query( + "UPDATE photos " + + "SET duplicate=:duplicate,modified=CURRENT_TIMESTAMP,scanned=CURRENT_TIMESTAMP WHERE id=:id", { + replacements: item, + transaction: t + }); + }); + }) + + setStatus("Looking for removed assets"); + let results = await photoDB.sequelize.query( + "SELECT photos.scanned,photos.id,photos.filename,albums.path FROM photos " + + "LEFT JOIN albums ON (albums.id=photos.albumId) " + + "WHERE photos.deleted=0 AND (DATETIME(photos.scanned) { + const onDisk = await exists(item.path + item.filename); + if (!onDisk) { + setStatus( + `${item.path}${item.filename} no longer exists on disk. ` + + `Marking as deleted.`); + deleted.push(item.id); + } + }); + + if (deleted.length) { + await photoDB.sequelize.transaction(async (t) => { + await photoDB.sequelize.query( + "UPDATE photos SET deleted=1,scanned=CURRENT_TIMESTAMP " + + "WHERE id IN (:deleted)", { + replacements: { deleted }, + transaction: t + } + ); + await photoDB.sequelize.query( + "DELETE FROM photohashes WHERE photoId IN (:deleted)", { + replacements: { deleted }, + transaction: t + } + ); + }); + } + + setStatus(`${deleted.length} assets deleted.`); + + processRunning = false; } function scanDir(parent, path) { @@ -435,7 +468,9 @@ function scanDir(parent, path) { return stat(filepath).then(function(stats) { if (stats.isDirectory()) { filepath += "/"; - return scanDir(album, filepath).spread(function(_albums, _assets) { + return scanDir(album, filepath).then((res) => { + const _albums = res.albums, + _assets = res.assets; album.allAssetCount += _assets.length; album.allAlbumCount += _albums.length + 1; albums = albums.concat(_albums); @@ -475,11 +510,11 @@ function scanDir(parent, path) { } }); }).then(function() { - return [ albums, assets ]; + return { albums, assets }; }); } -function findOrCreateDBAlbum(transaction, album) { +const findOrCreateDBAlbum = async (t, album) => { let query = "SELECT id FROM albums WHERE path=:path AND "; if (!album.parent) { query += "parentId IS NULL"; @@ -494,30 +529,29 @@ function findOrCreateDBAlbum(transaction, album) { query += "parentId=:parentId"; } - return photoDB.sequelize.query(query, { + const results = await photoDB.sequelize.query(query, { replacements: album, type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(results) { - if (results.length == 0) { - if (!album.parent) { - setStatus("Creating top level album: " + picturesPath, "warn" ); - } - return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", { - replacements: album, - transaction: transaction - }).then(array => { - return array[1].lastID; - }); - } else { - return results[0].id; - } - }).then(function(id) { - album.id = id; - return id; }); + + if (results.length == 0) { + if (!album.parent) { + setStatus("Creating top level album: " + picturesPath, "warn" ); + } + return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", { + replacements: album, + transaction: t + }).then(array => { + album.id = array[1].lastID; + }); + } else { + album.id = results[0].id; + } + + return album.id; } -const findOrUpdateDBAsset = async (transaction, asset) => { +const findOrUpdateDBAsset = async (t, asset) => { if (!asset.album || !asset.album.id) { let error = "Asset being processed without an album"; setStatus(error, "warn"); @@ -538,7 +572,7 @@ const findOrUpdateDBAsset = async (transaction, asset) => { return await photoDB.sequelize.query("INSERT INTO photos " + "(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", { replacements: asset, - transaction: transaction + transaction: t }).then(array => { asset.id = array[1].lastID; return asset; @@ -615,7 +649,7 @@ function setStatus(status, level) { } } -function doScan() { +const doScan = async () => { /* 1. Scan for all assets which will be managed by the system. readdir * 2. Check if entry in DB. Check mod-time in DB vs. stats from #1 * - For albums @@ -636,119 +670,156 @@ function doScan() { let needsProcessing = []; if (scanningStatus.length != 0) { - return Promise.resolve(scanningStatus); + return scanningStatus; } - return scanDir(null, picturesPath).spread(function(albums, assets) { - setStatus("Found " + assets.length + " assets in " + albums.length + " albums after " + - ((Date.now() - now) / 1000) + "s"); - /* One at a time, in series, as the album[] array has parents first, then descendants. - * Operating in parallel could result in a child being searched for prior to the parent */ - now = Date.now(); + let { albums, assets } = await scanDir(null, picturesPath); - let toProcess = albums.length, lastMessage = moment(); - return photoDB.sequelize.transaction(function(transaction) { - return Promise.mapSeries(albums, function(album) { - return findOrCreateDBAlbum(transaction, album).then(function() { - toProcess--; - if (moment().add(-5, 'seconds') > lastMessage) { - setStatus("Albums to be created in DB: " + toProcess); - lastMessage = moment(); - } - }); - }); - }).then(function() { - setStatus("Processed " + albums.length + " album DB entries in " + - ((Date.now() - now) / 1000) + "s"); - now = Date.now(); + setStatus( + `Found ${assets.length} assets in ${albums.length} albums after ` + + `${(Date.now() - now) / 1000}s`); + + /* One at a time, in series, as the album[] array has parents + * first, then descendants. + * Operating in parallel could result in a child being searched for + * prior to the parent */ + now = Date.now(); - setStatus(assets.length + " assets remaining to be verified/updated. ETA N/A"); - - let processed = 0, start = Date.now(), last = 0, updateScanned = [], newEntries = 0; - return photoDB.sequelize.transaction(function(transaction) { - return Promise.map(assets, function(asset) { - return Promise.resolve(asset).then(function(asset) { - /* If both mtime and ctime of the asset are older than the lastScan, skip it - * - * Can only do this after a full scan has occurred */ - if (lastScan != null && asset.stats.mtime < lastScan && asset.stats.ctime < lastScan) { - return asset; - } - - return findOrUpdateDBAsset(transaction, asset).then(function(asset) { - if (!asset.scanned) { - newEntries++; - } - if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) { -// if (!asset.scanned) { console.log("no scan date on asset"); } -// if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); } -// if (!asset.modified) { console.log("no mtime."); } - needsProcessing.push(asset); - } else { - updateScanned.push(asset.id); - } - return asset; - }).then(function(asset) { - return asset; - }); - }).then(function(asset) { - processed++; - - let elapsed = Date.now() - start; - if (elapsed < 5000) { - return asset; - } - - let remaining = assets.length - processed, eta = Math.ceil((elapsed / 1000) * remaining / (processed - last)); - setStatus(remaining + " assets remaining be verified/updated " + - "(" + newEntries + " new entries, " + needsProcessing.length + " need processing," + (processed - newEntries) + " up-to-date so far). ETA " + eta + "s"); - last = processed; - start = Date.now(); - }); - }, { - concurrency: 10 - }); - }).then(function() { - if (updateScanned.length) { - return photoDB.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:ids)", { - replacements: { - ids: updateScanned - } - }).then(function() { - setStatus("Updated scan date of " + updateScanned.length + " assets"); - updateScanned = []; - }); + try { + let toProcess = albums.length, + lastMessage = moment(); + await photoDB.sequelize.transaction(async (t) => { + await Promise.mapSeries(albums, async (album) => { + await findOrCreateDBAlbum(t, album); + + toProcess--; + + if (moment().add(-5, 'seconds') > lastMessage) { + setStatus(`Albums to be created in DB: ${toProcess}`); + lastMessage = moment(); } - }).then(function() { - setStatus(newEntries + " assets are new. " + (needsProcessing.length - newEntries) + " assets have been modified.\n" + - needsProcessing.length + " assets need HASH computed. " + (assets.length - needsProcessing.length) + " need no update.");; - processBlock(needsProcessing); - needsProcessing = []; - }).then(function() { - setStatus("Scanned " + assets.length + " asset DB entries in " + - ((Date.now() - now) / 1000) + "s"); - assets = []; }); }); - }).then(function() { - setStatus("Total time to initialize DB and all scans: " + ((Date.now() - initialized) / 1000) + "s"); - return photoDB.sequelize.query("SELECT max(scanned) AS scanned FROM photos", { - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(results) { - if (results[0].scanned == null) { - lastScan = new Date("1800-01-01"); - } else { - lastScan = new Date(results[0].scanned); - } - setStatus("Updating any asset newer than " + moment(lastScan).format()); + } catch (error) { + console.error(error); + process.exit(-1); + } + + setStatus( + `Processed ${albums.length} album DB entries in ` + + `${(Date.now() - now) / 1000}s`); + + now = Date.now(); + + setStatus( + `${assets.length} assets remaining to be verified/updated. ETA N/A`); + + let updateScanned = [], + newEntries = 0; + + try { + let processed = 0, + start = Date.now(), + last = 0; + + await photoDB.sequelize.transaction(async (t) => { + await Promise.map(assets, async (asset) => { + /* If both mtime and ctime of the asset are older than the + * lastScan, skip it + * Can only do this after a full scan has occurred */ + if (lastScan != null + && asset.stats.mtime < lastScan + && asset.stats.ctime < lastScan) { + return; + } + + asset = await findOrUpdateDBAsset(t, asset); + if (!asset.scanned) { + newEntries++; + } + + if (!asset.scanned + || asset.scanned < asset.stats.mtime + || !asset.modified) { + // if (!asset.scanned) { console.log("no scan date on asset"); } + // if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); } + // if (!asset.modified) { console.log("no mtime."); } + needsProcessing.push(asset); + } else { + updateScanned.push(asset.id); + } + + processed++; + + let elapsed = Date.now() - start; + if (elapsed < 5000) { + return; + } + + let remaining = assets.length - processed, + eta = Math.ceil((elapsed / 1000) * remaining / (processed - last)); + setStatus( + `${remaining} assets remaining be verified/updated (${newEntries} ` + + `new entries, ${needsProcessing.length} need processing, ` + + `${(processed - newEntries)} up-to-date so far). ETA ${eta}s` + ); + last = processed; + start = Date.now(); + }, { + concurrency: 10 + }); }); - }).then(function() { - setStatus("idle"); - return "scan complete"; - }).catch(function(error) { - setStatus(error); - throw error; + } catch (error) { + console.error(error); + process.exit(-1); + } + + if (updateScanned.length) { + await photoDB.sequelize.query( + "UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:ids)", { + replacements: { + ids: updateScanned + } + }); + setStatus("Updated scan date of " + updateScanned.length + " assets"); + updateScanned = []; + } + + setStatus( + `${newEntries} assets are new. ` + + `${needsProcessing.length - newEntries} assets have been ` + + `modified.\n${needsProcessing.length} assets need HASH computed. ` + + `${assets.length - needsProcessing.length} need no update.`); + + processBlock(needsProcessing); + + needsProcessing = []; + + setStatus( + `Scanned ${assets.length} asset DB entries in ` + + `${(Date.now() - now) / 1000}s`); + assets = []; + + setStatus( + `Total time to initialize DB and all scans: ` + + `${(Date.now() - initialized) / 1000}s`); + + let results = await photoDB.sequelize.query( + "SELECT max(scanned) AS scanned FROM photos", { + type: photoDB.sequelize.QueryTypes.SELECT }); + + if (results[0].scanned == null) { + lastScan = new Date("1800-01-01"); + } else { + lastScan = new Date(results[0].scanned); + } + + setStatus(`Updating any asset newer than ${moment(lastScan).format()}`); + + setStatus("idle"); + + return "scan complete"; } module.exports = {