Compare commits
No commits in common. "6d536a75f9e57bf52e54348f75101af481b002f5" and "3e6bb96ab8ba419c7edf771252861ceba74a89c1" have entirely different histories.
6d536a75f9
...
3e6bb96ab8
@ -18,4 +18,3 @@ services:
|
||||
- ${PWD}/config/local.json:/website/config/local.json
|
||||
- /opt/ketrface/models:/root/.deepface
|
||||
# - ${PWD}:/website
|
||||
- ${PWD}/server:/website/server
|
||||
|
||||
@ -138,9 +138,8 @@ const determineImageDate = (asset, metadata) => {
|
||||
const created = asset.stats.mtime,
|
||||
filename = asset.filename;
|
||||
|
||||
|
||||
/* Attempt to find CREATED / MODIFIED date based on meta-data or
|
||||
* FILENAME patterns */
|
||||
* FILENAME patterns */
|
||||
if (metadata.exif
|
||||
&& metadata.exif.exif
|
||||
&& metadata.exif.exif.DateTimeOriginal
|
||||
@ -202,9 +201,9 @@ const processImageAsset = async (asset) => {
|
||||
filename = asset.filename;
|
||||
|
||||
let src = picturesPath + path + filename,
|
||||
image = sharp(src, { limitInputPixels: false });
|
||||
image = sharp(src);
|
||||
|
||||
const metadata = await image
|
||||
const metadata = await image/*.limitInputPixels(1073741824)*/
|
||||
.metadata()
|
||||
.catch(error => console.error(error) );
|
||||
|
||||
@ -283,13 +282,9 @@ const processBlock = async (items) => {
|
||||
|
||||
let toProcess = processing.length, lastMessage = moment();
|
||||
|
||||
setStatus(`Items to be processed: ${toProcess}`);
|
||||
setStatus("Items to be processed: " + toProcess);
|
||||
|
||||
const updateHash = [],
|
||||
insertHash = [],
|
||||
updateAsset = [];
|
||||
|
||||
await Promise.map(processing, async (asset) => {
|
||||
await Promise.mapSeries(processing, async (asset) => {
|
||||
toProcess--;
|
||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
||||
setStatus("Items to be processed: " + toProcess);
|
||||
@ -323,9 +318,9 @@ const processBlock = async (items) => {
|
||||
let query;
|
||||
|
||||
if (results.length == 0) {
|
||||
insertHash.push(asset);
|
||||
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
|
||||
} else if (results[0].hash != asset.hash) {
|
||||
updateHash.push(asset);
|
||||
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
|
||||
} else if (results[0].photoId != asset.id) {
|
||||
setStatus("Duplicate asset: " +
|
||||
"'" + asset.album.path + asset.filename + "' is a copy of " +
|
||||
@ -350,7 +345,7 @@ const processBlock = async (items) => {
|
||||
needsProcessing.push(asset);
|
||||
|
||||
try {
|
||||
await processImageAsset(asset); /* no DB writes */
|
||||
await processImageAsset(asset)
|
||||
} catch (error) {
|
||||
const path = asset.album.path,
|
||||
filename = asset.filename;
|
||||
@ -361,41 +356,13 @@ const processBlock = async (items) => {
|
||||
return;
|
||||
}
|
||||
|
||||
updateAsset.push(asset);
|
||||
}, {
|
||||
concurrency: require('os').cpus().length
|
||||
});
|
||||
|
||||
try {
|
||||
await photoDB.sequelize.transaction(async (t) => {
|
||||
await Promise.mapSeries(updateHash, async (item) => {
|
||||
await photoDB.sequelize.query(
|
||||
"UPDATE photohashes SET hash=:hash WHERE photoId=:id", {
|
||||
replacements: item,
|
||||
transaction: t
|
||||
});
|
||||
});
|
||||
await Promise.mapSeries(insertHash, async (item) => {
|
||||
await photoDB.sequelize.query(
|
||||
"INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)", {
|
||||
replacements: item,
|
||||
transaction: t
|
||||
});
|
||||
});
|
||||
/* Update the DB with the image information */
|
||||
await Promise.mapSeries(updateAsset, async (item) => {
|
||||
await photoDB.sequelize.query("UPDATE photos SET " +
|
||||
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " +
|
||||
"WHERE id=:id", {
|
||||
replacements: item,
|
||||
transaction: t
|
||||
});
|
||||
});
|
||||
/* Update the DB with the image information */
|
||||
await photoDB.sequelize.query("UPDATE photos SET " +
|
||||
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " +
|
||||
"WHERE id=:id", {
|
||||
replacements: asset,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exit(-1);
|
||||
}
|
||||
});
|
||||
|
||||
/* Process the DUPLICATES */
|
||||
setStatus(
|
||||
@ -584,19 +551,7 @@ const findOrCreateDBAlbum = async (t, album) => {
|
||||
return album.id;
|
||||
}
|
||||
|
||||
const createDBAsset = async (asset, transaction) => {
|
||||
return photoDB.sequelize.query(
|
||||
"INSERT INTO photos " +
|
||||
"(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", {
|
||||
replacements: asset,
|
||||
transaction
|
||||
}).then(array => {
|
||||
asset.id = array[1].lastID;
|
||||
return asset;
|
||||
});
|
||||
}
|
||||
|
||||
const findDBAsset = async (asset) => {
|
||||
const findOrUpdateDBAsset = async (asset) => {
|
||||
if (!asset.album || !asset.album.id) {
|
||||
let error = "Asset being processed without an album";
|
||||
setStatus(error, "warn");
|
||||
@ -614,7 +569,14 @@ const findDBAsset = async (asset) => {
|
||||
});
|
||||
|
||||
if (results.length == 0) {
|
||||
return undefined;
|
||||
return photoDB.sequelize.query(
|
||||
"INSERT INTO photos " +
|
||||
"(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", {
|
||||
replacements: asset
|
||||
}).then(array => {
|
||||
asset.id = array[1].lastID;
|
||||
return asset;
|
||||
});
|
||||
}
|
||||
|
||||
asset.id = results[0].id;
|
||||
@ -625,9 +587,7 @@ const findDBAsset = async (asset) => {
|
||||
* shouldn't happen in production unless someone modifies the file, then
|
||||
* re-stamps the modified time */
|
||||
if (asset.size != results[0].size) {
|
||||
setStatus(
|
||||
`File was modified with time-restamp (HASH regeneration will be ` +
|
||||
`queued): ${asset.filename}`);
|
||||
setStatus("File was modified with time-restamp (HASH regeneration will be queued): " + asset.filename);
|
||||
delete asset.scanned;
|
||||
delete asset.modified;
|
||||
}
|
||||
@ -731,7 +691,6 @@ const doScan = async () => {
|
||||
lastMessage = moment();
|
||||
await photoDB.sequelize.transaction(async (t) => {
|
||||
await Promise.mapSeries(albums, async (album) => {
|
||||
|
||||
await findOrCreateDBAlbum(t, album);
|
||||
|
||||
toProcess--;
|
||||
@ -757,15 +716,14 @@ const doScan = async () => {
|
||||
`${assets.length} assets remaining to be verified/updated. ETA N/A`);
|
||||
|
||||
let updateScanned = [],
|
||||
newEntries = 0,
|
||||
needsCreation = [];
|
||||
newEntries = 0;
|
||||
|
||||
try {
|
||||
let processed = 0,
|
||||
start = Date.now(),
|
||||
last = 0;
|
||||
|
||||
await Promise.map(assets, async (asset) => {
|
||||
await Promise.mapSeries(assets, async (asset) => {
|
||||
/* If both mtime and ctime of the asset are older than the
|
||||
* lastScan, skip it
|
||||
* Can only do this after a full scan has occurred */
|
||||
@ -775,11 +733,7 @@ const doScan = async () => {
|
||||
return;
|
||||
}
|
||||
|
||||
const res = await findDBAsset(asset);
|
||||
if (!res) {
|
||||
needsCreation.push(asset);
|
||||
}
|
||||
|
||||
asset = await findOrUpdateDBAsset(asset);
|
||||
if (!asset.scanned) {
|
||||
newEntries++;
|
||||
}
|
||||
@ -805,27 +759,15 @@ const doScan = async () => {
|
||||
let remaining = assets.length - processed,
|
||||
eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
|
||||
setStatus(
|
||||
`${remaining} assets remaining be verified/updated (${newEntries} ` +
|
||||
`${remaining} assets remaining to be verified/updated (${newEntries} ` +
|
||||
`new entries, ${needsProcessing.length} need processing, ` +
|
||||
`${(processed - newEntries)} up-to-date so far). ETA ${eta}s`
|
||||
);
|
||||
last = processed;
|
||||
start = Date.now();
|
||||
} , {
|
||||
concurrency: 10
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
try {
|
||||
console.log(`Creating ${needsCreation.length} asset entries.`);
|
||||
await photoDB.sequelize.transaction(async (t) => {
|
||||
await Promise.mapSeries(needsCreation, async (asset) => {
|
||||
await createDBAsset(asset, t);
|
||||
});
|
||||
});
|
||||
/*, {
|
||||
concurrency: 10
|
||||
*/});
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exit(-1);
|
||||
@ -845,7 +787,7 @@ const doScan = async () => {
|
||||
setStatus(
|
||||
`${newEntries} assets are new. ` +
|
||||
`${needsProcessing.length - newEntries} assets have been ` +
|
||||
`modified.\n${needsProcessing.length} assets need to be processed. ` +
|
||||
`modified.\n${needsProcessing.length} assets need HASH computed. ` +
|
||||
`${assets.length - needsProcessing.length} need no update.`);
|
||||
|
||||
processBlock(needsProcessing);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user