jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Album and photo loading is all messed up

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2018-09-27 00:30:15 -07:00
parent e9dc145812
commit d3c08b5389
5 changed files with 92 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
frontend/elements/photo-thumbnail.html
View File

@ -101,7 +101,7 @@
if (item === undefined|| base === undefined || item.path === undefined) {
return "";
}
return base + item.path + "/thumbs/" + item.filename;
return base + item.path + "thumbs/" + item.filename;
},
date: function(item) {
@ -124,9 +124,9 @@
attached: function() {
var base = document.querySelector("base");
if (base) {
this.base = new URL(base.href).pathname.replace(/\/$/, ""); /* Remove trailing slash if there */
this.base = new URL(base.href).pathname.replace(/\/$/, "") + "/"; /* Ensure trailing slash */
} else {
this.base = "";
this.base = "/";
}
}
});

29
frontend/src/ketr-photos/ketr-photos.html
View File

@ -32,7 +32,6 @@
<link rel="import" href="../../elements/photo-lightbox.html">
<link rel="import" href="../../elements/photo-thumbnail.html">
<link rel="import" href="../../elements/pan-line.html">
<script src="fetch.js"></script>
@ -382,7 +381,8 @@
</div>
</paper-dialog>
<paper-toast id="toast"></paper-toast>
<photo-lightbox tabindex="0" id="lightbox" on-close="lightBoxClose" on-next="lightBoxNext" on-previous="lightBoxPrevious"></photo-lightbox>
<photo-lightbox tabindex="0"
id="lightbox" on-close="lightBoxClose" on-next="lightBoxNext" on-previous="lightBoxPrevious"></photo-lightbox>
</template>
<script>
@ -878,7 +878,7 @@
}
this.$.lightbox.item = el.item;
this.$.lightbox.src = this.base + el.item.path + "/thumbs/scaled/" + el.item.filename;
this.$.lightbox.src = this.base + el.item.path + "thumbs/scaled/" + el.item.filename;
this.lightBoxElement = el;
this.lightBoxElement.selected = true;
this.disableScrolling = true;
@ -943,7 +943,12 @@
// thumbnail.width = this.calcWidth;
thumbnail.addEventListener("load-image", this._imageTap.bind(this));
thumbnail.addEventListener("load-album", this.loadAlbum.bind(this));
datetime = new Date((photo.taken || photo.modified || photo.added) + " GMT").toISOString().replace(/T.*$/, "");
try {
datetime = new Date((photo.taken || photo.modified || photo.added).replace(/T.*/, "") + " GMT").toISOString().replace(/T.*$/, "");
} catch (error) {
console.log(JSON.stringify(photo, null, 2));
throw error;
}
var dateBlock = this.root.querySelector("#date-" + datetime), thumbnails;
if (!dateBlock) {
@ -1043,7 +1048,7 @@
query += key + "=" + encodeURIComponent(params[key]);
}
var path = this.path || "", mode = this.mode;
var path = this.path || "/", mode = this.mode;
if (mode != "albums") {
path = "/" + mode;
if (mode == "time") {
@ -1053,7 +1058,7 @@
}
}
var username = this.user ? this.user.username : "";
console.log("Requesting " + this.limit + " photos.");
console.log("Requesting " + this.limit + " photos from " + path);
window.fetch("api/v1/photos" + path + query, function(path, error, xhr) {
this.loading = false;
@ -1063,7 +1068,7 @@
if ((username != (this.user ? this.user.username : "")) ||
(mode != this.mode) ||
((mode == "albums") && (path != (this.path || ""))) ||
((mode == "albums") && (path != (this.path || "/"))) ||
((mode == "memories") && (path != ("/memories/" + (this.date || ""))))) {
console.log("Skipping results for old query. Triggering re-fetch of photos for new path or mode.");
this._loadPhotos();
@ -1089,9 +1094,9 @@
var base = document.querySelector("base");
if (base) {
this.base = new URL(base.href).pathname.replace(/\/$/, ""); /* Remove trailing slash if there */
this.base = new URL(base.href).pathname.replace(/\/$/, "") + "/"; /* Make sure there is a trailing slash */
} else {
this.base = "";
this.base = "/";
}
console.log(results.items.length + " photos received.");
@ -1112,7 +1117,7 @@
}
this.loadingAlbums = true;
var path = this.path || "";
var path = this.path || "/";
window.fetch("api/v1/albums" + path, function(path, error, xhr) {
this.loadingAlbums = false;
@ -1120,14 +1125,14 @@
return;
}
if (path != (this.path || "")) {
if (path != (this.path || "/")) {
console.log("Skipping results for old query. Triggering re-fetch of albums for new path.");
this._loadAlbums();
return;
}
if (error) {
console.log("Error loading album: " + (this.path || ""));
console.log("Error loading album: " + (this.path || "/"));
console.error(JSON.stringify(error, null, 2));
return;
}

4
server/routes/albums.js
View File

@ -15,9 +15,9 @@ require("../db/photos").then(function(db) {
const router = express.Router();
router.get("/*", function(req, res/*, next*/) {
let url = decodeURI(req.url).replace(/\?.*$/, ""),
let url = decodeURI(req.url).replace(/\?.*$/, "").replace(/^\//, ""),
query = "SELECT * FROM albums WHERE path=:path";
console.log("Looking up album: " + url);
return photoDB.sequelize.query(query, {
replacements: {
path: url

8
server/routes/photos.js
View File

@ -40,11 +40,11 @@ router.get("/memories/*", function(req, res/*, next*/) {
if (id == -1) {
index = "";
} else {
index = " AND ((taken=DATE(:cursor) AND id<"+id+ ") OR taken<DATE(:cursor))";
index = " AND ((taken=DATE(:cursor) AND photos.id<"+id+ ") OR taken<DATE(:cursor))";
}
let date = new Date(decodeURI(req.url).replace(/\?.*$/, ""));
let query = "SELECT * FROM photos WHERE strftime('%m%d',taken)=strftime('%m%d',:date) " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
let query = "SELECT photos.*,albums.path AS path FROM photos INNER JOIN albums ON albums.id=photos.albumId WHERE strftime('%m%d',taken)=strftime('%m%d',:date) " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
// console.log("Memories for " + date.toISOString().replace(/T.*/, ""));
// console.log(query);
@ -109,11 +109,11 @@ router.get("/*", function(req, res/*, next*/) {
if (id == -1) {
index = "";
} else {
index = " AND ((taken=DATE(:cursor) AND id<"+id+ ") OR taken<DATE(:cursor))";
index = " AND ((taken=DATE(:cursor) AND photos.id<"+id+ ") OR taken<DATE(:cursor))";
}
let path = decodeURI(req.url).replace(/\?.*$/, ""),
query = "SELECT photos.* FROM photos INNER JOIN albums ON albums.id=photos.albumId AND albums.path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
query = "SELECT photos.*,albums.path AS path FROM photos INNER JOIN albums ON albums.id=photos.albumId AND albums.path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
console.log("Fetching from: " + path);

128
server/scanner.js
View File

@ -195,15 +195,52 @@ function processBlock(items) {
processQueue = processQueue.concat(items);
}
if (!processRunning && processQueue.length) {
let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0);
processRunning = true;
/* Sort to newest files to be processed first */
processing.sort(function(a, b) {
return a.stats.mtime - b.stats.mtime;
});
if (processRunning || processQueue.length == 0) {
return;
}
return Promise.map(processing, function(asset) {
let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0),
needsProcessing = [], duplicates = [];
processRunning = true;
/* Sort to newest files to be processed first */
processing.sort(function(a, b) {
return a.stats.mtime - b.stats.mtime;
});
return Promise.map(processing, function(asset) {
return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
asset.hash = hash;
});
}).then(function() {
/* Needs to be one at a time in case there are multiple HASH collisions */
return Promise.mapSeries(processing, function(asset) {
return photoDB.sequelize.query("SELECT * FROM photohashes WHERE hash=:hash OR photoId=:id", {
replacements: asset,
type: photoDB.sequelize.QueryTypes.SELECT
}).then(function(results) {
let query;
if (results.length == 0) {
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
} else if (results[0].hash != asset.hash) {
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
} else if (results[0].photoId != asset.id) {
console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename);
duplicates.push(asset);
return;
} else {
return;
}
return photoDB.sequelize.query(query, {
replacements: asset
}).then(function() {
/* HASH has been updated; add to the needsProcessing array */
needsProcessing.push(asset);
});
});
});
}).then(function() {
return Promise.map(needsProcessing, function(asset) {
var path = asset.album.path,
file = asset.filename,
created = asset.stats.ctime,
@ -308,8 +345,7 @@ function processBlock(items) {
});
}).then(function() {
return photoDB.sequelize.query("UPDATE photos SET " +
"(added,modified,taken,width,height,scanned)" +
"VALUES(:added,:modified,:taken,:width,:height,CURRENT_TIMESTAMP)", {
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP", {
replacements: asset
});
}).then(function() {
@ -349,10 +385,10 @@ function processBlock(items) {
});
}, {
concurrency: 1
}).then(function() {
console.log("Completed processing queue.");
});
}
}).then(function() {
console.log("Completed processing queue. " + duplicates.length + " duplicates.");
});
}
@ -360,7 +396,7 @@ function scanDir(parent, path) {
let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
album = {
path: path.slice(picturesPath.length), /* path already ends in '/' */
name: path.replace(/.*\//, "").replace(/_/g, " "),
name: path.replace(/\/$/, "").replace(/.*\//, "").replace(/_/g, " "),
parent: parent,
allAssetCount: 0,
allAlbumCount: 0
@ -458,7 +494,7 @@ function findOrCreateDBAlbum(album) {
}).then(function(results) {
if (results.length == 0) {
if (!album.parent) {
console.warn("Creating top level album: " + album.path);
console.warn("Creating top level album: " + picturesPath);
}
return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
replacements: album
@ -540,17 +576,20 @@ module.exports = {
* 2. Check if entry in DB. Check mod-time in DB vs. stats from #1
* - For albums
* - For assets
* 3. If not in DB, or mod-time changed, compute HASH of the file
* 4. Check for HASH in photohash -- skip?
* 5. Check for and create thumbs/FILE thumbs/scaled/FILE
* 6. If necessary, create JPG from RAW
* 7. Update last-scanned date in DB for entry
* 8. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were
* 3. If not in DB, or mod-time changed, queue for HASH CHECK
*
* HASH CHECK
* 1. Compute HASH
* 2. Check for HASH in photohash -- skip?
* 3. Check for and create thumbs/FILE thumbs/scaled/FILE
* 4. If necessary, create JPG from RAW
* 5. Update last-scanned date in DB for entry
* 6. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were
* removed on disk)? Also purge from the HASH table.
*/
let initialized = Date.now();
let now = Date.now();
const needsProcessing = [], duplicates = [];
const needsProcessing = [];
return scanDir(null, picturesPath).spread(function(albums, assets) {
console.log("Found " + assets.length + " assets in " + albums.length + " albums after " +
((Date.now() - now) / 1000) + "s");
@ -565,11 +604,11 @@ module.exports = {
((Date.now() - now) / 1000) + "s");
now = Date.now();
let processed = 0, start = Date.now(), last = 0, hashNeeded = [];
let processed = 0, start = Date.now(), last = 0;
return Promise.map(assets, function(asset) {
return findOrUpdateDBAsset(asset).then(function(asset) {
if (asset.scanned < asset.stats.mtime) {
hashNeeded.push(asset);
needsProcessing.push(asset);
}
}).then(function(asset) {
processed++;
@ -588,46 +627,11 @@ module.exports = {
}, {
concurrency: 5
}).then(function() {
console.log(hashNeeded.length + " assets need HASH computed");
return Promise.map(hashNeeded, function(asset) {
return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
asset.hash = hash;
});
}).then(function() {
/* Needs to be one at a time in case there are multiple HASH collisions */
return Promise.mapSeries(hashNeeded, function(asset) {
return db.sequelize.query("SELECT * FROM photohashes WHERE hash=:hash OR photoId=:id", {
replacements: asset,
type: photoDB.sequelize.QueryTypes.SELECT
}).then(function(results) {
let query;
if (results.length == 0) {
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
} else if (results[0].hash != asset.hash) {
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id)";
} else if (results[0].photoId != asset.id) {
console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename);
duplicates.push(asset);
return;
} else {
return;
}
return db.sequelize.query(query, {
replacements: asset
}).then(function() {
/* HASH has been updated; add to the needsProcessing array */
needsProcessing.push(asset);
});
});
}).then(function() {
processBlock(needsProcessing);
});
});
console.log(needsProcessing.length + " assets need HASH computed");
processBlock(needsProcessing);
}).then(function() {
console.log("Processed " + assets.length + " asset DB entries in " +
console.log("Scanned " + assets.length + " asset DB entries in " +
((Date.now() - now) / 1000) + "s");
console.log(duplicates.length + " duplicates.");
});
});
}).then(function() {