jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Restructured and removed transaction usage during update due to DB lockups

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2023-01-12 14:17:07 -08:00
parent 2d9aa1a9f6
commit 3e6bb96ab8
1 changed files with 48 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
server/scanner.js
View File

@ -551,7 +551,7 @@ const findOrCreateDBAlbum = async (t, album) => {
return album.id; return album.id;
} }
const findOrUpdateDBAsset = async (t, asset) => { const findOrUpdateDBAsset = async (asset) => {
if (!asset.album || !asset.album.id) { if (!asset.album || !asset.album.id) {
let error = "Asset being processed without an album"; let error = "Asset being processed without an album";
setStatus(error, "warn"); setStatus(error, "warn");
@ -569,10 +569,10 @@ const findOrUpdateDBAsset = async (t, asset) => {
}); });
if (results.length == 0) { if (results.length == 0) {
return await photoDB.sequelize.query("INSERT INTO photos " + return photoDB.sequelize.query(
"INSERT INTO photos " +
"(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", { "(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", {
replacements: asset, replacements: asset
transaction: t
}).then(array => { }).then(array => {
asset.id = array[1].lastID; asset.id = array[1].lastID;
return asset; return asset;
@ -583,8 +583,9 @@ const findOrUpdateDBAsset = async (t, asset) => {
asset.scanned = new Date(results[0].scanned); asset.scanned = new Date(results[0].scanned);
asset.modified = new Date(results[0].modified); asset.modified = new Date(results[0].modified);
/* If the size on disk changed, update the size entry in the DB. This shouldn't happen in /* If the size on disk changed, update the size entry in the DB. This
* production unless someone modifies the file, then re-stamps the modified time */ * shouldn't happen in production unless someone modifies the file, then
* re-stamps the modified time */
if (asset.size != results[0].size) { if (asset.size != results[0].size) {
setStatus("File was modified with time-restamp (HASH regeneration will be queued): " + asset.filename); setStatus("File was modified with time-restamp (HASH regeneration will be queued): " + asset.filename);
delete asset.scanned; delete asset.scanned;
@ -722,53 +723,51 @@ const doScan = async () => {
start = Date.now(), start = Date.now(),
last = 0; last = 0;
await photoDB.sequelize.transaction(async (t) => { await Promise.mapSeries(assets, async (asset) => {
await Promise.map(assets, async (asset) => { /* If both mtime and ctime of the asset are older than the
/* If both mtime and ctime of the asset are older than the * lastScan, skip it
* lastScan, skip it * Can only do this after a full scan has occurred */
* Can only do this after a full scan has occurred */ if (lastScan != null
if (lastScan != null && asset.stats.mtime < lastScan
&& asset.stats.mtime < lastScan && asset.stats.ctime < lastScan) {
&& asset.stats.ctime < lastScan) { return;
return; }
}
asset = await findOrUpdateDBAsset(t, asset); asset = await findOrUpdateDBAsset(asset);
if (!asset.scanned) { if (!asset.scanned) {
newEntries++; newEntries++;
} }
if (!asset.scanned if (!asset.scanned
|| asset.scanned < asset.stats.mtime || asset.scanned < asset.stats.mtime
|| !asset.modified) { || !asset.modified) {
// if (!asset.scanned) { console.log("no scan date on asset"); } // if (!asset.scanned) { console.log("no scan date on asset"); }
// if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); } // if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); }
// if (!asset.modified) { console.log("no mtime."); } // if (!asset.modified) { console.log("no mtime."); }
needsProcessing.push(asset); needsProcessing.push(asset);
} else { } else {
updateScanned.push(asset.id); updateScanned.push(asset.id);
} }
processed++; processed++;
let elapsed = Date.now() - start; let elapsed = Date.now() - start;
if (elapsed < 5000) { if (elapsed < 5000) {
return; return;
} }
let remaining = assets.length - processed, let remaining = assets.length - processed,
eta = Math.ceil((elapsed / 1000) * remaining / (processed - last)); eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
setStatus( setStatus(
`${remaining} assets remaining be verified/updated (${newEntries} ` + `${remaining} assets remaining to be verified/updated (${newEntries} ` +
`new entries, ${needsProcessing.length} need processing, ` + `new entries, ${needsProcessing.length} need processing, ` +
`${(processed - newEntries)} up-to-date so far). ETA ${eta}s` `${(processed - newEntries)} up-to-date so far). ETA ${eta}s`
); );
last = processed; last = processed;
start = Date.now(); start = Date.now();
}, { /*, {
concurrency: 10 concurrency: 10
}); */});
});
} catch (error) { } catch (error) {
console.error(error); console.error(error);
process.exit(-1); process.exit(-1);