jketreno/ketr.photos
1
0
Fork 0
Code Issues 1 Activity

Scanning and everything now works

Browse Source 
Signed-off-by: James Ketrenos <james_git@ketrenos.com>
This commit is contained in:
James Ketr 2018-09-27 23:30:21 -07:00
parent 698cd6a202
commit 383b5fae61
7 changed files with 71 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
frontend/src/ketr-photos/ketr-photos.html
View File

@ -655,11 +655,11 @@
},
breadcrumb: function(path) {
var crumbs = path.split("/"), parts = [];
var crumbs = path.replace(/(.*)\/$/, "/$1").split("/"), parts = [];
path = "";
crumbs.forEach(function(crumb, index) {
if (crumb) {
path += "/" + crumb;
if (index > 0) {
path += crumb + "/";
}
parts.push({
name: crumb ? crumb : "Top",
@ -736,7 +736,7 @@
if (top) {
var photo = top.item;
this.$.pager.style.opacity = 1;
var date = window.moment(new Date((photo.taken || photo.modified || photo.added) + " GMT"));
var date = window.moment(new Date((photo.taken || photo.modified || photo.added).replace(/T.*/, " GMT")));
date = date.calendar(null, { sameElse: "MMM DD, YYYY" }).replace(/ at.*/, "");
this.$.pager.textContent = date;
this.$.pager.style.top = headerHeight +
@ -966,11 +966,9 @@
return;
}
var i = 0;
photos.forEach(function(photo) {
var datetime = new Date((photo.taken || photo.modified || photo.added) + " GMT"),
year = datetime.getFullYear();
for (; i < this.years.length; i++) {
var year = (photo.taken || photo.modified || photo.added).replace(/^(....).*$/, "$1");
for (var i = 0; i < this.years.length; i++) {
if (this.years[i] == year) {
return;
}
@ -1008,7 +1006,7 @@
thumbnail.addEventListener("load-image", this._imageTap.bind(this));
thumbnail.addEventListener("load-album", this.loadAlbum.bind(this));
try {
datetime = new Date((photo.taken || photo.modified || photo.added).replace(/T.*/, "") + " GMT").toISOString().replace(/T.*$/, "");
datetime = (photo.taken || photo.modified || photo.added).replace(/T.*/, "");
} catch (error) {
console.log(JSON.stringify(photo, null, 2));
throw error;
@ -1112,18 +1110,18 @@
query += key + "=" + encodeURIComponent(params[key]);
}
var path = this.path || "/", mode = this.mode;
var path = this.path || "", mode = this.mode;
if (mode != "albums") {
path = "/" + mode;
path = mode;
if (mode == "time") {
path = "";
} else {
path = "/memories/" + (this.date || "");
path = "memories/" + (this.date || "");
}
}
var username = this.user ? this.user.username : "";
console.log("Requesting " + this.limit + " photos from " + path);
window.fetch("api/v1/photos" + path + query, function(path, error, xhr) {
window.fetch("api/v1/photos/" + path + query, function(path, error, xhr) {
this.loading = false;
if (!this.user) {
@ -1132,8 +1130,8 @@
if ((username != (this.user ? this.user.username : "")) ||
(mode != this.mode) ||
((mode == "albums") && (path != (this.path || "/"))) ||
((mode == "memories") && (path != ("/memories/" + (this.date || ""))))) {
((mode == "albums") && (path != (this.path || ""))) ||
((mode == "memories") && (path != ("memories/" + (this.date || ""))))) {
console.log("Skipping results for old query. Triggering re-fetch of photos for new path or mode.");
this._loadPhotos();
return;
@ -1181,22 +1179,22 @@
}
this.loadingAlbums = true;
var path = this.path || "/";
window.fetch("api/v1/albums" + path, function(path, error, xhr) {
var path = this.path || "";
window.fetch("api/v1/albums/" + path, function(path, error, xhr) {
this.loadingAlbums = false;
if (!this.user) {
return;
}
if (path != (this.path || "/")) {
if (path != (this.path || "")) {
console.log("Skipping results for old query. Triggering re-fetch of albums for new path.");
this._loadAlbums();
return;
}
if (error) {
console.log("Error loading album: " + (this.path || "/"));
console.log("Error loading album: " + (this.path || ""));
console.error(JSON.stringify(error, null, 2));
return;
}

5
server/app.js
View File

@ -41,8 +41,6 @@ app.set("trust proxy", true);
/* Handle static files first so excessive logging doesn't occur */
app.use(basePath, express.static("frontend", { index: false }));
app.use(morgan("common"));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: false
@ -109,6 +107,9 @@ app.use(basePath, function(req, res, next) {
});
app.use(basePath, express.static(picturesPath, { index: false }));
app.use(morgan("common"));
app.use(basePath + "api/v1/photos", require("./routes/photos"));
app.use(basePath + "api/v1/days", require("./routes/days"));
app.use(basePath + "api/v1/albums", require("./routes/albums"));

4
server/db/photos.js
View File

@ -60,6 +60,10 @@ function init() {
taken: Sequelize.DATE,
width: Sequelize.INTEGER,
height: Sequelize.INTEGER,
duplicate: {
type: Sequelize.BOOLEAN,
defaultValue: 0
},
albumId: {
type: Sequelize.INTEGER,
allowNull: true,

2
server/routes/albums.js
View File

@ -17,7 +17,7 @@ const router = express.Router();
router.get("/*", function(req, res/*, next*/) {
let url = decodeURI(req.url).replace(/\?.*$/, "").replace(/^\//, ""),
query = "SELECT * FROM albums WHERE path=:path";
console.log("Looking up album: " + url);
// console.log("Looking up album: " + url);
return photoDB.sequelize.query(query, {
replacements: {
path: url

27
server/routes/photos.js
View File

@ -38,13 +38,15 @@ router.get("/memories/*", function(req, res/*, next*/) {
}
if (id == -1) {
index = "";
index = "strftime('%m%d',taken)=strftime('%m%d',:date)";
} else {
index = " AND ((taken=DATE(:cursor) AND photos.id<"+id+ ") OR taken<DATE(:cursor))";
index = "(strftime('%m%d',taken)=strftime('%m%d',:date) AND photos.id<"+id+ ")";
}
let date = new Date(decodeURI(req.url).replace(/\?.*$/, ""));
let query = "SELECT photos.*,albums.path AS path FROM photos INNER JOIN albums ON albums.id=photos.albumId WHERE strftime('%m%d',taken)=strftime('%m%d',:date) " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
let query = "SELECT photos.*,albums.path AS path FROM photos " +
"INNER JOIN albums ON (albums.id=photos.albumId) " +
"WHERE (photos.duplicate=0 AND photos.scanned NOT NULL AND " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
// console.log("Memories for " + date.toISOString().replace(/T.*/, ""));
// console.log(query);
@ -109,19 +111,22 @@ router.get("/*", function(req, res/*, next*/) {
if (id == -1) {
index = "";
} else {
index = " AND ((taken=DATE(:cursor) AND photos.id<"+id+ ") OR taken<DATE(:cursor))";
index = "AND ((strftime('%m%d',taken)=strftime('%m%d',:cursor) AND photos.id<"+id+ ") OR DATE(taken)<DATE(:cursor))";
}
let path = decodeURI(req.url).replace(/\?.*$/, ""),
query = "SELECT photos.*,albums.path AS path FROM photos INNER JOIN albums ON albums.id=photos.albumId AND albums.path LIKE :path " + index + " ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1);
let path = decodeURI(req.url).replace(/\?.*$/, "").replace(/^\//, ""),
query = "SELECT photos.*,albums.path AS path FROM photos " +
"INNER JOIN albums ON (albums.id=photos.albumId AND albums.path LIKE :path) " +
"WHERE (photos.duplicate=0 AND photos.scanned NOT NULL " + index + ") ORDER BY taken DESC,id DESC LIMIT " + (limit * 2 + 1),
replacements = {
cursor: cursor,
path: path + "%"
};
console.log("Fetching from: " + path);
// console.log("Fetching from: " + JSON.stringify(replacements, null, 2) + "\nwith:\n" + query);
return photoDB.sequelize.query(query, {
replacements: {
cursor: cursor,
path: path + "%"
},
replacements: replacements,
type: photoDB.Sequelize.QueryTypes.SELECT
}).then(function(photos) {
photos.forEach(function(photo) {

3
server/routes/users.js
View File

@ -101,7 +101,8 @@ router.post("/login", function(req, res) {
* found there, we look them up in the site-specific user database */
return ldapPromise(username, password).then(function(user) {
return user;
user.authenticated = 1;
return user;
}).catch(function() {
console.log("User not found in LDAP. Looking up in DB.");
let query = "SELECT * FROM users WHERE uid=:username AND password=:password";

34
server/scanner.js
View File

@ -192,6 +192,7 @@ function processBlock(items) {
setTimeout(processBlock, 1000);
if (items) {
console.log("Adding " + items.length + " on top of " + processQueue.length + " assets.");
processQueue = processQueue.concat(items);
}
@ -201,7 +202,9 @@ function processBlock(items) {
let lastMessage = moment(), toProcess = processQueue.length, processing = processQueue.splice(0),
needsProcessing = [], duplicates = [];
processRunning = true;
/* Sort to newest files to be processed first */
processing.sort(function(a, b) {
return a.stats.mtime - b.stats.mtime;
@ -227,19 +230,22 @@ function processBlock(items) {
console.log("Duplicate asset: " + asset.id + " vs " + results[0].photoId + ". Skipping " + asset.album.path + asset.filename);
duplicates.push(asset);
return;
} else {
}
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
needsProcessing.push(asset);
if (!query) {
return;
}
return photoDB.sequelize.query(query, {
replacements: asset
}).then(function() {
/* HASH has been updated; add to the needsProcessing array */
needsProcessing.push(asset);
});
});
});
}).then(function() {
console.log(needsProcessing.length + " assets need to have metadata extracted");
return Promise.map(needsProcessing, function(asset) {
var path = asset.album.path,
file = asset.filename,
@ -343,7 +349,8 @@ function processBlock(items) {
});
}).then(function() {
return photoDB.sequelize.query("UPDATE photos SET " +
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP", {
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,scanned=CURRENT_TIMESTAMP " +
"WHERE id=:id", {
replacements: asset
});
}).then(function() {
@ -385,7 +392,22 @@ function processBlock(items) {
concurrency: 1
});
}).then(function() {
console.log("Completed processing queue. " + duplicates.length + " duplicates.");
console.log("Completed processing queue. Marking " + duplicates.length + " duplicates.");
let dups = [];
duplicates.forEach(function(asset) {
/* If not already marked as a duplicate, mark it. */
if (!asset.duplicate) {
dups.push(asset.id);
}
});
return photoDB.sequelize.query("UPDATE photos SET duplicate=1,scanned=CURRENT_TIME WHERE id IN (:dups)", {
replacements: {
dups: dups
}
});
}).then(function() {
processRunning = false;
});
}