jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Restructured and removed transaction usage during update due to DB lockups

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2023-01-12 14:17:07 -08:00
parent 2d9aa1a9f6
commit 3e6bb96ab8
1 changed files with 48 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
server/scanner.js
View File

@ -137,7 +137,7 @@ function moveCorrupt(path, file) {
const determineImageDate = (asset, metadata) => {
const created = asset.stats.mtime,
filename = asset.filename;
/* Attempt to find CREATED / MODIFIED date based on meta-data or
* FILENAME patterns */
if (metadata.exif
@ -551,7 +551,7 @@ const findOrCreateDBAlbum = async (t, album) => {
return album.id;
}
const findOrUpdateDBAsset = async (t, asset) => {
const findOrUpdateDBAsset = async (asset) => {
if (!asset.album || !asset.album.id) {
let error = "Asset being processed without an album";
setStatus(error, "warn");
@ -569,10 +569,10 @@ const findOrUpdateDBAsset = async (t, asset) => {
});
if (results.length == 0) {
return await photoDB.sequelize.query("INSERT INTO photos " +
return photoDB.sequelize.query(
"INSERT INTO photos " +
"(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", {
replacements: asset,
transaction: t
replacements: asset
}).then(array => {
asset.id = array[1].lastID;
return asset;
@ -583,8 +583,9 @@ const findOrUpdateDBAsset = async (t, asset) => {
asset.scanned = new Date(results[0].scanned);
asset.modified = new Date(results[0].modified);
/* If the size on disk changed, update the size entry in the DB. This shouldn't happen in
* production unless someone modifies the file, then re-stamps the modified time */
/* If the size on disk changed, update the size entry in the DB. This
* shouldn't happen in production unless someone modifies the file, then
* re-stamps the modified time */
if (asset.size != results[0].size) {
setStatus("File was modified with time-restamp (HASH regeneration will be queued): " + asset.filename);
delete asset.scanned;
@ -722,53 +723,51 @@ const doScan = async () => {
start = Date.now(),
last = 0;
await photoDB.sequelize.transaction(async (t) => {
await Promise.map(assets, async (asset) => {
/* If both mtime and ctime of the asset are older than the
* lastScan, skip it
* Can only do this after a full scan has occurred */
if (lastScan != null
&& asset.stats.mtime < lastScan
&& asset.stats.ctime < lastScan) {
return;
}
await Promise.mapSeries(assets, async (asset) => {
/* If both mtime and ctime of the asset are older than the
* lastScan, skip it
* Can only do this after a full scan has occurred */
if (lastScan != null
&& asset.stats.mtime < lastScan
&& asset.stats.ctime < lastScan) {
return;
}
asset = await findOrUpdateDBAsset(t, asset);
if (!asset.scanned) {
newEntries++;
}
if (!asset.scanned
|| asset.scanned < asset.stats.mtime
|| !asset.modified) {
// if (!asset.scanned) { console.log("no scan date on asset"); }
// if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); }
// if (!asset.modified) { console.log("no mtime."); }
needsProcessing.push(asset);
} else {
updateScanned.push(asset.id);
}
asset = await findOrUpdateDBAsset(asset);
if (!asset.scanned) {
newEntries++;
}
processed++;
if (!asset.scanned
|| asset.scanned < asset.stats.mtime
|| !asset.modified) {
// if (!asset.scanned) { console.log("no scan date on asset"); }
// if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); }
// if (!asset.modified) { console.log("no mtime."); }
needsProcessing.push(asset);
} else {
updateScanned.push(asset.id);
}
processed++;
let elapsed = Date.now() - start;
if (elapsed < 5000) {
return;
}
let elapsed = Date.now() - start;
if (elapsed < 5000) {
return;
}
let remaining = assets.length - processed,
eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
setStatus(
`${remaining} assets remaining be verified/updated (${newEntries} ` +
`new entries, ${needsProcessing.length} need processing, ` +
`${(processed - newEntries)} up-to-date so far). ETA ${eta}s`
);
last = processed;
start = Date.now();
}, {
let remaining = assets.length - processed,
eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
setStatus(
`${remaining} assets remaining to be verified/updated (${newEntries} ` +
`new entries, ${needsProcessing.length} need processing, ` +
`${(processed - newEntries)} up-to-date so far). ETA ${eta}s`
);
last = processed;
start = Date.now();
/*, {
concurrency: 10
});
});
*/});
} catch (error) {
console.error(error);
process.exit(-1);