Fixed concurrency on hash creation.
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
parent
4f1d72147a
commit
e0bf69faef
@ -209,51 +209,51 @@ function processBlock(items) {
|
|||||||
return asset;
|
return asset;
|
||||||
}).then(function(asset) {
|
}).then(function(asset) {
|
||||||
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
|
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
|
||||||
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
|
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
|
||||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
||||||
"WHERE hash=:hash OR photoId=:id", {
|
"WHERE hash=:hash OR photoId=:id", {
|
||||||
replacements: asset,
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT
|
|
||||||
}).then(function(results) {
|
|
||||||
let query;
|
|
||||||
|
|
||||||
if (results.length == 0) {
|
|
||||||
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
|
|
||||||
} else if (results[0].hash != asset.hash) {
|
|
||||||
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
|
|
||||||
} else if (results[0].photoId != asset.id) {
|
|
||||||
console.log("Duplicate asset: " +
|
|
||||||
"'" + asset.album.path + asset.filename + "' is a copy of " +
|
|
||||||
"'" + results[0].path + results[0].filename + "'");
|
|
||||||
duplicates.push(asset);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
|
|
||||||
needsProcessing.push(asset);
|
|
||||||
|
|
||||||
if (!query) {
|
|
||||||
return asset;
|
|
||||||
}
|
|
||||||
|
|
||||||
return photoDB.sequelize.query(query, {
|
|
||||||
replacements: asset,
|
replacements: asset,
|
||||||
|
type: photoDB.sequelize.QueryTypes.SELECT
|
||||||
|
}).then(function(results) {
|
||||||
|
let query;
|
||||||
|
|
||||||
|
if (results.length == 0) {
|
||||||
|
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
|
||||||
|
} else if (results[0].hash != asset.hash) {
|
||||||
|
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
|
||||||
|
} else if (results[0].photoId != asset.id) {
|
||||||
|
console.log("Duplicate asset: " +
|
||||||
|
"'" + asset.album.path + asset.filename + "' is a copy of " +
|
||||||
|
"'" + results[0].path + results[0].filename + "'");
|
||||||
|
duplicates.push(asset);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
|
||||||
|
needsProcessing.push(asset);
|
||||||
|
|
||||||
|
if (!query) {
|
||||||
|
return asset;
|
||||||
|
}
|
||||||
|
|
||||||
|
return photoDB.sequelize.query(query, {
|
||||||
|
replacements: asset,
|
||||||
|
}).then(function() {
|
||||||
|
return asset;
|
||||||
|
});
|
||||||
}).then(function() {
|
}).then(function() {
|
||||||
return asset;
|
toProcess--;
|
||||||
|
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||||
|
console.log("Hash items to be processed: " + toProcess);
|
||||||
|
lastMessage = moment();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}).then(function() {
|
|
||||||
toProcess--;
|
|
||||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
|
||||||
console.log("Hash items to be processed: " + toProcess);
|
|
||||||
lastMessage = moment();
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}, {
|
}, {
|
||||||
concurrency: 5
|
concurrency: 5
|
||||||
}).catch(function(error) {
|
}).catch(function(error) {
|
||||||
console.log("Error commiting HASH transactions");
|
console.log("Error commiting HASH transactions");
|
||||||
throw error;
|
throw error;
|
||||||
});
|
|
||||||
}).then(function() {
|
}).then(function() {
|
||||||
let toProcess = needsProcessing.length, lastMessage = moment();
|
let toProcess = needsProcessing.length, lastMessage = moment();
|
||||||
console.log(needsProcessing.length + " assets need to have metadata extracted");
|
console.log(needsProcessing.length + " assets need to have metadata extracted");
|
||||||
@ -682,7 +682,7 @@ module.exports = {
|
|||||||
((Date.now() - now) / 1000) + "s");
|
((Date.now() - now) / 1000) + "s");
|
||||||
now = Date.now();
|
now = Date.now();
|
||||||
|
|
||||||
console.log(albums.length + " assets remaining to have DB entries updated. ETA N/A");
|
console.log(assets.length + " assets remaining to have DB entries updated. ETA N/A");
|
||||||
|
|
||||||
let processed = 0, start = Date.now(), last = 0, updateScanned = [];
|
let processed = 0, start = Date.now(), last = 0, updateScanned = [];
|
||||||
return photoDB.sequelize.transaction(function(transaction) {
|
return photoDB.sequelize.transaction(function(transaction) {
|
||||||
@ -693,22 +693,22 @@ module.exports = {
|
|||||||
} else {
|
} else {
|
||||||
updateScanned.push(asset.id);
|
updateScanned.push(asset.id);
|
||||||
}
|
}
|
||||||
|
}).then(function(asset) {
|
||||||
|
processed++;
|
||||||
|
|
||||||
|
let elapsed = Date.now() - start;
|
||||||
|
if (elapsed < 5000) {
|
||||||
|
return asset;
|
||||||
|
}
|
||||||
|
|
||||||
|
let remaining = assets.length - processed;
|
||||||
|
console.log(remaining + " assets remaining to have DB entries updated. ETA " +
|
||||||
|
Math.ceil((elapsed / 1000) * remaining / (processed - last)) + "s");
|
||||||
|
last = processed;
|
||||||
|
start = Date.now();
|
||||||
});
|
});
|
||||||
}, {
|
}, {
|
||||||
concurrency: 10
|
concurrency: 10
|
||||||
}).then(function(asset) {
|
|
||||||
processed++;
|
|
||||||
|
|
||||||
let elapsed = Date.now() - start;
|
|
||||||
if (elapsed < 5000) {
|
|
||||||
return asset;
|
|
||||||
}
|
|
||||||
|
|
||||||
let remaining = assets.length - processed;
|
|
||||||
console.log(remaining + " assets remaining to have DB entries updated. ETA " +
|
|
||||||
Math.ceil((elapsed / 1000) * remaining / (processed - last)) + "s");
|
|
||||||
last = processed;
|
|
||||||
start = Date.now();
|
|
||||||
});
|
});
|
||||||
}).then(function() {
|
}).then(function() {
|
||||||
if (updateScanned.length) {
|
if (updateScanned.length) {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user