docker-compose.yml

@@ -18,4 +18,3 @@ services:
       - ${PWD}/config/local.json:/website/config/local.json
       - /opt/ketrface/models:/root/.deepface
 #      - ${PWD}:/website
-      - ${PWD}/server:/website/server

server/scanner.js

@@ -138,9 +138,8 @@ const determineImageDate = (asset, metadata) => {
   const created = asset.stats.mtime,
     filename = asset.filename;
 
-
   /* Attempt to find CREATED / MODIFIED date based on meta-data or
-  * FILENAME patterns */
+       * FILENAME patterns */
   if (metadata.exif
     && metadata.exif.exif
     && metadata.exif.exif.DateTimeOriginal
@@ -202,9 +201,9 @@ const processImageAsset = async (asset) => {
     filename = asset.filename;
 
   let src = picturesPath + path + filename,
-    image = sharp(src, { limitInputPixels: false });
+    image = sharp(src);
   
-  const metadata = await image
+  const metadata = await image/*.limitInputPixels(1073741824)*/
     .metadata()
     .catch(error => console.error(error) );
   
@@ -283,13 +282,9 @@ const processBlock = async (items) => {
 
   let toProcess = processing.length, lastMessage = moment();
 
-  setStatus(`Items to be processed: ${toProcess}`);
+  setStatus("Items to be processed: " + toProcess);
 
-  const updateHash = [],
+  await Promise.mapSeries(processing, async (asset) => {
-    insertHash = [],
-    updateAsset = [];
-    
-  await Promise.map(processing, async (asset) => {
     toProcess--;
     if (moment().add(-5, 'seconds') > lastMessage) {
       setStatus("Items to be processed: " + toProcess);
@@ -323,9 +318,9 @@ const processBlock = async (items) => {
     let query;
 
     if (results.length == 0) {
-      insertHash.push(asset);
+      query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
     } else if (results[0].hash != asset.hash) {
-      updateHash.push(asset);
+      query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
     } else if (results[0].photoId != asset.id) {
       setStatus("Duplicate asset: " + 
         "'" + asset.album.path + asset.filename + "' is a copy of " +
@@ -350,7 +345,7 @@ const processBlock = async (items) => {
     needsProcessing.push(asset);
 
     try {
-      await processImageAsset(asset); /* no DB writes */
+      await processImageAsset(asset)
     } catch (error) {
       const path = asset.album.path,
         filename = asset.filename;
@@ -361,41 +356,13 @@ const processBlock = async (items) => {
       return;
     }
 
-    updateAsset.push(asset);
+    /* Update the DB with the image information */
-  }, {
+    await photoDB.sequelize.query("UPDATE photos SET " +
-    concurrency: require('os').cpus().length
+      "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " +
-  });
+      "WHERE id=:id", {
-
+      replacements: asset,
-  try {
-    await photoDB.sequelize.transaction(async (t) => {
-      await Promise.mapSeries(updateHash, async (item) => {
-        await photoDB.sequelize.query(
-          "UPDATE photohashes SET hash=:hash WHERE photoId=:id", {
-            replacements: item,
-            transaction: t
-        });
-      });
-      await Promise.mapSeries(insertHash, async (item) => {
-        await photoDB.sequelize.query(
-          "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)", {
-            replacements: item,
-            transaction: t
-        });
-      });
-      /* Update the DB with the image information */
-      await Promise.mapSeries(updateAsset, async (item) => {
-        await photoDB.sequelize.query("UPDATE photos SET " +
-          "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " +
-          "WHERE id=:id", {
-          replacements: item,
-          transaction: t
-        });
-      });
     });
-  } catch (error) {
+  });
-    console.error(error);
-    process.exit(-1);
-  }
 
   /* Process the DUPLICATES */
   setStatus(
@@ -584,19 +551,7 @@ const findOrCreateDBAlbum = async (t, album) => {
   return album.id;
 }
 
-const createDBAsset = async (asset, transaction) => {
+const findOrUpdateDBAsset = async (asset) => {
-  return photoDB.sequelize.query(
-    "INSERT INTO photos " +
-    "(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", {
-    replacements: asset,
-    transaction
-  }).then(array => {
-    asset.id = array[1].lastID;
-    return asset;
-  });
-}
-
-const findDBAsset = async (asset) => {
   if (!asset.album || !asset.album.id) {
     let error = "Asset being processed without an album";
     setStatus(error, "warn");
@@ -614,7 +569,14 @@ const findDBAsset = async (asset) => {
   });
   
   if (results.length == 0) {
-    return undefined;
+    return photoDB.sequelize.query(
+      "INSERT INTO photos " +
+      "(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", {
+      replacements: asset
+    }).then(array => {
+      asset.id = array[1].lastID;
+      return asset;
+    });
   }
   
   asset.id = results[0].id;
@@ -625,9 +587,7 @@ const findDBAsset = async (asset) => {
    * shouldn't happen in production unless someone modifies the file, then 
    * re-stamps the modified time */
   if (asset.size != results[0].size) {
-    setStatus(
+    setStatus("File was modified with time-restamp (HASH regeneration will be queued): " + asset.filename);
-      `File was modified with time-restamp (HASH regeneration will be ` +
-      `queued): ${asset.filename}`);
     delete asset.scanned;
     delete asset.modified;
   }
@@ -731,7 +691,6 @@ const doScan = async () => {
       lastMessage = moment();
     await photoDB.sequelize.transaction(async (t) => {
       await Promise.mapSeries(albums, async (album) => {
-        
         await findOrCreateDBAlbum(t, album);
         
         toProcess--;
@@ -757,15 +716,14 @@ const doScan = async () => {
     `${assets.length} assets remaining to be verified/updated. ETA N/A`);
 
   let updateScanned = [],
-    newEntries = 0,
+    newEntries = 0;
-    needsCreation = [];
 
   try {
     let processed = 0, 
       start = Date.now(), 
       last = 0;
      
-    await Promise.map(assets, async (asset) => {
+    await Promise.mapSeries(assets, async (asset) => {
       /* If both mtime and ctime of the asset are older than the 
         * lastScan, skip it 
         * Can only do this after a full scan has occurred */
@@ -775,11 +733,7 @@ const doScan = async () => {
         return;
       }
 
-      const res = await findDBAsset(asset);
+      asset = await findOrUpdateDBAsset(asset);
-      if (!res) {
-        needsCreation.push(asset);
-      }
-
       if (!asset.scanned) {
         newEntries++;
       }
@@ -805,27 +759,15 @@ const doScan = async () => {
       let remaining = assets.length - processed, 
         eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
       setStatus(
-        `${remaining} assets remaining be verified/updated (${newEntries} ` +
+        `${remaining} assets remaining to be verified/updated (${newEntries} ` +
         `new entries, ${needsProcessing.length} need processing, ` +
         `${(processed - newEntries)} up-to-date so far). ETA ${eta}s`
       );
       last = processed;
       start = Date.now();
-    } , {
+    /*, {
-      concurrency: 10
+        concurrency: 10
-    });
+    */});
-  } catch (error) {
-    console.error(error);
-    process.exit(-1);
-  }
-
-  try {
-    console.log(`Creating ${needsCreation.length} asset entries.`);
-    await photoDB.sequelize.transaction(async (t) => {
-      await Promise.mapSeries(needsCreation, async (asset) => {
-        await createDBAsset(asset, t);
-      });
-    });
   } catch (error) {
     console.error(error);
     process.exit(-1);
@@ -845,7 +787,7 @@ const doScan = async () => {
   setStatus(
     `${newEntries} assets are new. ` +
     `${needsProcessing.length - newEntries} assets have been ` +
-    `modified.\n${needsProcessing.length} assets need to be processed. ` +
+    `modified.\n${needsProcessing.length} assets need HASH computed. ` +
     `${assets.length - needsProcessing.length} need no update.`);
  
   processBlock(needsProcessing);