From 94776ca2339c19f104f29238f99d6073a0798907 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Thu, 9 Jan 2020 21:37:00 -0800 Subject: [PATCH] Advanced filtering Signed-off-by: James Ketrenos --- clusters-pre | 14 ++++- frontend/face-explorer.html | 14 +++-- scanner/scanner.c | 115 +++++++++++++++++++++++++++++------- server/routes/photos.js | 73 ++++++++++------------- 4 files changed, 144 insertions(+), 72 deletions(-) diff --git a/clusters-pre b/clusters-pre index 941e5c0..91ebbed 100644 --- a/clusters-pre +++ b/clusters-pre @@ -9,15 +9,23 @@ document.addEventListener("DOMContentLoaded", (event) => { var div = document.createElement("div"); div.textContent = "Cluster " + (index + 1) + " has " + cluster.length + " neighbors."; document.body.appendChild(div); - cluster.forEach((id, index) => { + cluster.forEach((tuple, index) => { if (index > 15) { return; } var div = document.createElement("div"); div.classList.add("face"); - div.style.backgroundImage = "url(face-data/" + (id % 100) + "/" + id + "-original.png)"; + let faceId = tuple[0], + photoId = tuple[1]; + div.setAttribute("photo-id", photoId); + div.style.backgroundImage = "url(face-data/" + (faceId % 100) + "/" + faceId + "-original.png)"; div.addEventListener("click", (event) => { - window.open("face-explorer.html?" + id, "photo-" + id); + let photoId = parseInt(event.currentTarget.getAttribute("photo-id")); + if (photoId) { + window.open("face-explorer.html?" + photoId, "photo-" + photoId); + } else { + alert("No photo id mapped to face."); + } }); document.body.appendChild(div); }); diff --git a/frontend/face-explorer.html b/frontend/face-explorer.html index ab7b544..cdcdca3 100644 --- a/frontend/face-explorer.html +++ b/frontend/face-explorer.html @@ -44,11 +44,14 @@ body { border-radius: 0.5em; opacity: 0.5; cursor: pointer; + text-align: center; + color: rgba(0, 0, 0, 0); } .face:hover { border-color: #ff0000; background-color: rgba(128,0,0,0.5); + color: red; } #photo { @@ -152,6 +155,7 @@ function makeFaceBoxes() { box.style.top = offsetTop + face.top * height + "%"; box.style.width = ((face.right - face.left) * width) + "%"; box.style.height = ((face.bottom - face.top) * height) + "%"; + box.textContent = face.faceConfidence.toFixed(2); box.addEventListener("click", (event) => { setPause(true); @@ -167,19 +171,19 @@ function makeFaceBoxes() { } } - if (face.relatedPhotos.length == 0) { + if (face.relatedFaces.length == 0) { document.body.removeChild(bar); } - face.relatedPhotos.forEach((photo) => { + face.relatedFaces.forEach((related) => { var view = document.createElement("div"), - id = photo.faceId, + id = related.faceId, url = base + "face-data/" + (id % 100) + "/" + id + "-original.png" view.classList.add("view"); view.style.backgroundImage = "url(" + url + ")"; - view.textContent = photo.distance.toFixed(2); + view.textContent = related.faceConfidence.toFixed(2) + ":" +related.distance.toFixed(2); view.addEventListener("click", (event) => { - window.location.search = photo.id; + window.location.search = related.photoId; event.preventDefault = true; event.stopImmediatePropagation(); event.stopPropagation(); diff --git a/scanner/scanner.c b/scanner/scanner.c index 6567bf1..79c2756 100644 --- a/scanner/scanner.c +++ b/scanner/scanner.c @@ -25,6 +25,8 @@ typedef struct Face { long double descriptor[128]; long int clusterId; long faceId; + long photoId; + double confidence; ClusterTypes clusterType; double *distances; } Face; @@ -123,11 +125,15 @@ FaceLink *RangeQuery(Face **ppFaces, long int faceCount, Face *pQ, double eps) { FaceLink *pNeighbors = NULL; for (long int i = 0; i < faceCount; i++) { Face *pFace = ppFaces[i]; + if (pFace->confidence <= 0.9) { + continue; + } + if (pFace->faceId == pQ->faceId) { continue; } - if (pQ->distances[i] <= eps) { + if (pQ->distances[i] > 0.0 && pQ->distances[i] <= eps) { FaceLink *pLink = malloc(sizeof(*pLink)); memset(pLink, 0, sizeof(*pLink)); pLink->pFace = pFace; @@ -161,6 +167,10 @@ long int DBSCAN(Face **ppFaces, long int faceCount, double eps, int minPts) { long int C = 0; for (long int i = 0; i < faceCount; i++) { Face *pFace = ppFaces[i]; + if (pFace->confidence <= 0.9) { + continue; + } + if (pFace->clusterType != UNDEFINED) { continue; } @@ -230,6 +240,27 @@ long int DBSCAN(Face **ppFaces, long int faceCount, double eps, int minPts) { return C; } +typedef struct { + Face **ppFaces; + long int count; +} FaceCallbackData; + +int parseFaceIdRow(void *data, int argc, char **argv, char **column) { + FaceCallbackData *map = data; + long int faceId = strtol(argv[0] ? argv[0] : "0", NULL, 10); + long int photoId = strtol(argv[1] ? argv[1] : "0", NULL, 10); + double confidence = strtod(argv[2] ? argv[2] : "0.0", NULL); + for (long int i = 0; i < map->count; i++) { + if (map->ppFaces[i]->faceId == faceId) { + map->ppFaces[i]->photoId = photoId; + map->ppFaces[i]->confidence = confidence; + break; + } + } + return 0; +} + + /* * 1. Count how many entries there are * 2. Allocate storage to hold all entries @@ -241,7 +272,24 @@ int main(int argc, char *argv[]) { long maxId = 0; long i; long entries = 0; + long int minPts = MIN_PTS; + long double maxDistance = MAX_DISTANCE; + + if (argc == 1) { + fprintf(stderr, "usage: scanner PATH MAX_DISTANCE MIN_PTS\n"); + return -1; + } + if (argc > 2) { + sscanf(argv[2], "%Lf", &maxDistance); + } + + if (argc > 3) { + sscanf(argv[3], "%ld", &minPts); + } + + fprintf(stderr, "\nmaxDistance : %Lf\nminPts : %ld\n", maxDistance, minPts); + for (i = 0; i < 100; i++) { sprintf(pathBuf, "%s/face-data/%ld", argv[1], i); DIR *faceDir = opendir(pathBuf); @@ -312,7 +360,7 @@ int main(int argc, char *argv[]) { if (processed % 1000 == 0) { int perc = 100 * processed / (entries * entries); if (perc != last) { - fprintf(stderr, "Read %d%% of descriptors.\n", perc); + fprintf(stderr, "\rRead %d%% of descriptors.", perc); last = perc; } } @@ -320,7 +368,35 @@ int main(int argc, char *argv[]) { closedir(faceDir); } - fprintf(stderr, "Read %ld face descriptors...\n", entries); + fprintf(stderr, "\nRead %ld face descriptors...\n", entries); + + /* Allocate storage for all distances */ + sqlite3 *db; + + int rc = sqlite3_open("db/photos.db", &db); + if (rc != SQLITE_OK) { + fprintf(stderr, "Cannot open database: %s\n", sqlite3_errmsg(db)); + sqlite3_close(db); + return 1; + } + + fprintf(stderr, "DB opened."); + + char *err_msg = NULL; + FaceCallbackData data = { + ppFaces: ppFaces, + count: entries + }; + + rc = sqlite3_exec(db, "SELECT id,photoId,faceConfidence FROM faces", parseFaceIdRow, &data, &err_msg); + if (rc != SQLITE_OK) { + fprintf(stderr, "SQL error: %s\n", err_msg); + sqlite3_free(err_msg); + sqlite3_close(db); + return 1; + } + + fprintf(stderr, "Face data loaded from DB\n"); processed = 0; long double total = 0.0; @@ -332,10 +408,17 @@ int main(int argc, char *argv[]) { if (processed % 1000 == 0) { int perc = 100 * processed / (entries * entries); if (perc != last) { - fprintf(stderr, "Computed %d%% complete.\n", perc); + fprintf(stderr, "\rComputed %d%% complete.", perc); last = perc; } } + + if (pLink->confidence <= 0.9 || pTarget->confidence <= 0.9) { + pLink->distances[i] = 0.0; + pTarget->distances[j] = 0.0; + continue; + } + if (i == j) { pLink->distances[i] = 0.0; pTarget->distances[j] = 0.0; @@ -353,9 +436,9 @@ int main(int argc, char *argv[]) { } } - fprintf(stderr, "Average distance: %Lf\n", 1. * total / (entries * entries)); + fprintf(stderr, "\nAverage distance: %Lf\n", 1. * total / (entries * entries)); - long int clusters = DBSCAN(ppFaces, entries, MAX_DISTANCE, MIN_PTS); + long int clusters = DBSCAN(ppFaces, entries, maxDistance, minPts); long int undefined = 0, outlier = 0, core = 0, reachable = 0; for (i = 0; i < entries; i++) { switch (ppFaces[i]->clusterType) { @@ -387,9 +470,9 @@ int main(int argc, char *argv[]) { for (long int j = 0; j < entries; j++) { if (ppFaces[j]->clusterId == i) { if (nodes == 0) { - fprintf(stdout, "%ld", ppFaces[j]->faceId); + fprintf(stdout, "[%ld,%ld]", ppFaces[j]->faceId, ppFaces[j]->photoId); } else { - fprintf(stdout, ",%ld", ppFaces[j]->faceId); + fprintf(stdout, ",[%ld,%ld]", ppFaces[j]->faceId, ppFaces[j]->photoId); } nodes++; } @@ -402,19 +485,7 @@ int main(int argc, char *argv[]) { } fprintf(stdout, "];\n\n"); - /* Allocate storage for all distances */ - sqlite3 *db; - int rc = sqlite3_open("db/photos.db", &db); - if (rc != SQLITE_OK) { - fprintf(stderr, "Cannot open database: %s\n", sqlite3_errmsg(db)); - sqlite3_close(db); - return 1; - } - - fprintf(stderr, "DB opened."); - - char *err_msg = NULL; char *sql = "DELETE FROM facedistances;" "BEGIN TRANSACTION;"; @@ -445,7 +516,7 @@ int main(int argc, char *argv[]) { if (processed % 1000 == 0) { int perc = 100 * processed / (entries * entries); if (perc != last) { - fprintf(stderr, "Computed %d%% complete.\n", perc); + fprintf(stderr, "\rComputed %d%% complete.", perc); last = perc; } } @@ -479,6 +550,8 @@ int main(int argc, char *argv[]) { } } + fprintf(stderr, "\n"); + sprintf(sqlBuf, "UPDATE faces SET lastComparedId=%ld;", maxId); rc = sqlite3_exec(db, "COMMIT;", 0, 0, &err_msg); diff --git a/server/routes/photos.js b/server/routes/photos.js index dd28958..c60998b 100755 --- a/server/routes/photos.js +++ b/server/routes/photos.js @@ -780,62 +780,49 @@ router.get("/trash", function(req, res/*, next*/) { }); }); }); - + function getFacesForPhoto(id) { - return photoDB.sequelize.query("SELECT * FROM faces WHERE photoId=:id", { + /* Get the set of faces in this photo */ + return photoDB.sequelize.query( + "SELECT * FROM faces WHERE photoId=:id AND faceConfidence>0.9", { replacements: { - id: id + id: id, }, type: photoDB.Sequelize.QueryTypes.SELECT, raw: true }).then((faces) => { - return Promise.map(faces, (face) => { - return photoDB.sequelize.query( - "SELECT face1Id,face2Id " + - "FROM facedistances " + - "WHERE distance<=0.5 AND (face1Id=:id OR face2Id=:id)", { - replacements: { - id: face.id - }, - type: photoDB.Sequelize.QueryTypes.SELECT, - raw: true - }).then((faceIds) => { - return photoDB.sequelize.query( - "SELECT photos.id,faces.id AS faceId,fd.distance,albums.path,photos.filename " + - "FROM faces " + - "INNER JOIN photos ON photos.id=faces.photoId " + - "INNER JOIN albums ON albums.id=photos.albumId " + - "INNER JOIN facedistances AS fd ON " + - "( " + - " (fd.face1Id=faces.id AND fd.face2Id=:faceId) " + - "OR (fd.face2Id=faces.id AND fd.face1Id=:faceId) " + - ") " + - "WHERE faces.id IN (:ids) " + - "ORDER BY fd.distance ASC", { - replacements: { - ids: faceIds.map((match) => { - return (match.face1Id == face.id) ? match.face2Id : match.face1Id; - }), - faceId: face.id - }, - type: photoDB.Sequelize.QueryTypes.SELECT, - raw: true - }); - }).then((photos) => { - face.relatedPhotos = photos.filter((photo) => { return photo.id != id }).map((photo) => { + /* For each face in the photo, get the related faces */ + return photoDB.sequelize.query( + "SELECT relatedFaces.photoId AS photoId,fd.face1Id,fd.face2Id,fd.distance,relatedFaces.faceConfidence " + + "FROM (SELECT id,photoId,faceConfidence FROM faces WHERE faces.faceConfidence>=0.9 AND faces.id IN (:ids)) AS faces " + + "INNER JOIN faces AS relatedFaces ON relatedFaces.faceConfidence>=0.9 AND relatedFaces.id IN (fd.face1Id,fd.face2Id) " + + "INNER JOIN facedistances AS fd ON fd.distance<=0.5 " + + " AND (fd.face1Id=faces.id OR fd.face2Id=faces.id) " + + "WHERE (faces.id=fd.face1Id OR faces.id=fd.face2Id) " + + "ORDER BY fd.distance ASC", { + replacements: { + ids: faces.map(face => face.id), + }, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }).then((relatedFaces) => { + faces.forEach((face) => { + face.relatedFaces = relatedFaces.filter((related) => { + return (related.photoId != id && (related.face1Id == face.id || related.face2Id == face.id)); + }).map((related) => { return { - id: photo.id, - distance: photo.distance, - faceId: photo.faceId, - path: photo.path + photo.filename - }; + distance: related.distance, + faceConfidence: related.faceConfidence, + photoId: related.photoId, + faceId: related.face1Id != face.id ? related.face1Id : related.face2Id + } }); }); - }).then(() => { return faces; }); }); } + router.get("/faces/:id", (req, res) => { const id = parseInt(req.params.id); if (id != req.params.id) {