diff --git a/docker-compose.yml b/docker-compose.yml index 2b7967e..78c5384 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: # - db restart: always ports: - - ${PORT}:443 # nginx -> server/app.js express app + - ${PORT}:80 # nginx -> server/app.js express app # - 127.0.0.1:${SHELL_PORT}:4200 # shellinabox volumes: - /etc/letsencrypt:/etc/letsencrypt:ro # Use host web keys diff --git a/entrypoint.sh b/entrypoint.sh index 0877504..2f7cd4b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -30,6 +30,9 @@ if [[ -z "${DEVELOPMENT}" ]]; then { while true; do npm start ; sleep 3 ; done ; } else echo "Running in DEVELOPMENT mode." + if [[ ! -d /website/frontend ]]; then + fail "/website/frontend not found. Is the volume mounted? Did you run via './launch.sh'?" + fi if [[ ! -d /website/frontend/bower_components ]]; then echo "...installing bower_components for frontend" cd /website/frontend diff --git a/ketrface/cluster.py b/ketrface/cluster.py index e269896..70b783b 100644 --- a/ketrface/cluster.py +++ b/ketrface/cluster.py @@ -226,5 +226,7 @@ with conn: for identity in reduced: print(f'Writing identity {identity["id"]} to DB') id = create_identity(conn, identity) + first = True for face in identity['faces']: - update_face_identity(conn, face['id'], id) + update_face_identity(conn, face['id'], id, first) + first = False diff --git a/ketrface/ketrface/db.py b/ketrface/ketrface/db.py index 02544ed..f247a07 100644 --- a/ketrface/ketrface/db.py +++ b/ketrface/ketrface/db.py @@ -142,7 +142,7 @@ def create_identity(conn, identity): conn.commit() return cur.lastrowid -def update_face_identity(conn, faceId, identityId = None): +def update_face_identity(conn, faceId, identityId = None, first = False): """ Update the identity associated with this face :param conn: @@ -152,8 +152,13 @@ def update_face_identity(conn, faceId, identityId = None): """ sql = ''' UPDATE faces SET identityId=? WHERE id=? - ''' + ''' cur = conn.cursor() cur.execute(sql, (identityId, faceId)) + if first: + sql = ''' + UPDATE identities SET faceId=? WHERE id=? + ''' + cur.execute(sql, (faceId, identityId)) conn.commit() return None diff --git a/scripts/kill-client.sh b/scripts/kill-client.sh index 57af02e..a0401ad 100755 --- a/scripts/kill-client.sh +++ b/scripts/kill-client.sh @@ -1,7 +1,7 @@ #!/bin/bash pid=$(ps aux | - grep -E '[0-9] (/usr/bin/)?node .*client.*react-scripts/scripts/start.js' | + grep -E '[0-9] (/usr/bin/)?node .*client.*craco.*scripts/start.js' | while read user pid rest; do echo $pid; done) diff --git a/server/db/photos.js b/server/db/photos.js index 4a5922c..564758a 100755 --- a/server/db/photos.js +++ b/server/db/photos.js @@ -111,6 +111,14 @@ function init() { type: Sequelize.STRING, allowNull: false }, + faceId: { + type: Sequelize.INTEGER, + sallowNull: true, + references: { + model: Photo, + key: 'id', + } + }, descriptors: Sequelize.BLOB /* average of all faces mapped to this */ }, { timestamps: false @@ -180,6 +188,10 @@ function init() { key: 'id', } }, + distance: { + type: Sequelize.FLOAT, + defaultValue: -1 + }, classifiedBy: { type: Sequelize.DataTypes.ENUM( 'machine', diff --git a/server/development.location b/server/development.location index b46bb92..4f904a5 100644 --- a/server/development.location +++ b/server/development.location @@ -1,7 +1,7 @@ # DEVELOPMENT -- use npm development server on port 3000 (entrypoint.sh) location /identities/api/v1/ { rewrite ^/identities/api/v1/(.*)$ /api/v1/$1 break; - proxy_pass https://localhost/; + proxy_pass http://localhost/; proxy_redirect off; proxy_set_header Host $host; } @@ -17,5 +17,5 @@ location /identities { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; - proxy_pass https://localhost:3000; + proxy_pass http://localhost:3000; } diff --git a/server/nginx.conf b/server/nginx.conf index eef3b08..ed10d6d 100644 --- a/server/nginx.conf +++ b/server/nginx.conf @@ -1,17 +1,17 @@ server { listen 80 default_server; listen [::]:80 default_server; - return 301 https://$host$request_uri; -} +# return 301 https://$host$request_uri; +#} -server { - listen 443 ssl; +#server { +# listen 443 ssl; client_max_body_size 5g; - ssl on; +# ssl on; - ssl_certificate /etc/letsencrypt/live/ketrenos.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/ketrenos.com/privkey.pem; +# ssl_certificate /etc/letsencrypt/live/ketrenos.com/fullchain.pem; +# ssl_certificate_key /etc/letsencrypt/live/ketrenos.com/privkey.pem; root /website; index index.html; diff --git a/server/production.location b/server/production.location index 0d57dd9..dc140d9 100644 --- a/server/production.location +++ b/server/production.location @@ -1,10 +1,13 @@ # PRODUCTION -- pre-built source location /identities/api/v1/ { - rewrite ^/identities/api/v1/(.*)$ https://${host}/api/v1/$1 permanent; + rewrite ^/identities/api/v1/(.*)$ /api/v1/$1 break; + proxy_pass http://localhost/; + proxy_redirect off; + proxy_set_header Host $host; } location /identities { try_files $uri $uri/ =404; alias /website/client/build; index index.html; -} \ No newline at end of file +} diff --git a/server/routes/identities.js b/server/routes/identities.js index 4f52255..bd7625e 100755 --- a/server/routes/identities.js +++ b/server/routes/identities.js @@ -11,7 +11,7 @@ require("../db/photos").then(function(db) { const router = express.Router(); -const addOrUpdateIdentity = async(id, { +const upsertIdentity = async(id, { displayName, firstName, lastName, @@ -106,7 +106,7 @@ router.post('/', async (req, res) => { return res.status(401).send({ message: "Unauthorized to modify photos." }); } - const identity = await addOrUpdateIdentity(-1, req.body, res); + const identity = await upsertIdentity(-1, req.body, res); if (!identity) { return; } @@ -126,7 +126,7 @@ router.put('/:id', async (req, res) => { return res.status(400).send({ message: `Invalid identity id ${id}` }); } - const identity = await addOrUpdateIdentity(id, req.body, res); + const identity = await upsertIdentity(id, req.body, res); if (!identity) { return; } @@ -331,6 +331,106 @@ const getUnknownIdentity = async (faceCount) => { return unknownIdentity; } +/* Compute the identity's centroid descriptor from all faces + * and determine closest face to that centroid. If either of + * those values have changed, update the identity. + * + * Also updates the 'distance' on each face to the identity + * centroid. + */ +const updateIdentityFaces = async (identity) => { + if (!identity.identityId) { + identity.identityId = identity.id; + } + const faces = await photoDB.sequelize.query( + "SELECT " + + "faces.*,faceDescriptors.* " + + "WHERE " + + "faces.identityId=:identityId " + + "AND faceDescriptors.id=faces.descriptorsId", { + replacements: identity, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + }); + + let average = undefined, + closestId = -1, + closestDistance = -1, + count = 0; + + faces.forEach((face) => { + if (!descriptors[index]) { + return; + } + if (!face.descriptors) { + return; + } + face.distance = euclideanDistance( + face.descriptors, + identity.descriptors + ); + face.descriptors = bufferToFloat32Array(face.descriptors).map(x => x * x); + + if (closestId === -1) { + closestId = face.id; + closestDistance = face.distance; + average = descriptors; + count = 1; + return; + } + + descriptors.forEach((x, i) => { + average[i] += x; + }); + count++; + + if (face.distance < closestDistance) { + closestDistance = face.distance; + closestId = face.id; + } + }); + + let same = true; + if (average) { + average = average.map(x => x / count); + same = bufferToFloat32Array(identity.descriptors) + .find((x, i) => average[i] === x) === undefined; + await Promise(faces, async (face) => { + const distance = euclideanDistanceArray(face.descriptors, average); + if (distance !== face.distance) { + await photoDB.sequelize.query( + 'UPDATE faces SET distance=:distance WHERE id=:faceId', { + replacements: face + } + ); + } + }); + } + + let sql = ''; + if (closestId !== undefined && closestId !== identity.faceId) { + sql = `${sql} faceId=:faceId`; + } + if (!same) { + if (sql !== '') { + sql = `${sql}, `; + } + sql = `${sql} descriptors=:descriptors`; + } + if (sql !== '') { + await photoDB.sequelize.select( + `UPDATE identities SET ${sql} ` + + `WHERE id=:identityId`, { + replacements: { + faceId: closestId, + descriptors: average, + identityId: identity.id + } + } + ); + } +}; + router.get("/:id?", async (req, res) => { console.log(`GET ${req.url}`); @@ -352,100 +452,64 @@ router.get("/:id?", async (req, res) => { const filter = id ? "WHERE identities.id=:id " : ""; - const identities = await photoDB.sequelize.query("SELECT " + - "identities.*," + - "GROUP_CONCAT(faces.id) AS relatedFaceIds," + - "GROUP_CONCAT(faces.descriptorId) AS relatedFaceDescriptorIds," + - "GROUP_CONCAT(faces.photoId) AS relatedFacePhotoIds " + + const identities = await photoDB.sequelize.query( + "SELECT " + + "identities.id AS identityId," + + "identities.firstName," + + "identities.lastName," + + "identities.middleName," + + "identities.displayName," + + "identities.faceId " + "FROM identities " + - "LEFT JOIN faces ON identities.id=faces.identityId " + - filter + - "GROUP BY identities.id", { + filter, { replacements: { id }, type: photoDB.Sequelize.QueryTypes.SELECT, raw: true }); await Promise.map(identities, async (identity) => { - [ 'firstName', 'middleName', 'lastName' ].forEach(key => { - if (!identity[key]) { - identity[key] = ''; + let where; + /* If id was set, only return a single face */ + if (id !== undefined) { + if (identity.faceId !== -1) { + where = 'faceId=:faceId'; + } else { + where = 'identityId=:identityId LIMIT 1'; } - }); - identity.identityId = identity.id; - - if (!identity.relatedFaceIds) { - identity.relatedFaces = []; } else { - const relatedFaces = identity.relatedFaceIds.split(","), - relatedFacePhotos = identity.relatedFacePhotoIds.split(","); + where = 'identityId=:identityId' + } + identity.relatedFaces = await photoDB.sequelize.query( + 'SELECT id as faceId,photoId,distance FROM faces ' + + `WHERE ${where} ORDER BY distance ASC`, { + replacements: identity, + type: photoDB.Sequelize.QueryTypes.SELECT, + raw: true + } + ); - let descriptors = await photoDB.sequelize.query( - `SELECT descriptors FROM facedescriptors WHERE id in (:ids)`, { - replacements: { - ids: identity.relatedFaceDescriptorIds.split(',') - }, - type: photoDB.Sequelize.QueryTypes.SELECT, - raw: true - } - ); - - descriptors = descriptors.map(entry => entry.descriptors); - identity.relatedFaces = relatedFaces.map((faceId, index) => { - let distance = 0; - if (descriptors[index] && identity.descriptors) { - distance = euclideanDistance( - descriptors[index], - identity.descriptors - ); - } else { - distance = -1; - } - - return { - identityId: identity.id, - faceId, - photoId: relatedFacePhotos[index], - distance - }; - }); + if (identity.relatedFaces.length !== 0 + && (!identity.faceId || identity.faceId === -1)) { + await updateIdentityFaces(identity); } + /* If there were no faces, then add a 'Unknown' face */ if (identity.relatedFaces.length === 0) { identity.relatedFaces.push({ faceId: -1, photoId: -1, - identityId: identity.id, + identityId: identity.identityId, distance: 0, faceConfidence: 0 }); } - - identity - .relatedFaces - .sort((A, B) => { - return A.distance - B.distance; - }); - - /* If no filter was specified, only return the best face for - * the identity */ - if (!filter) { - identity.relatedFaces = [ identity.relatedFaces[0] ]; - } - - delete identity.id; - delete identity.descriptors; - delete identity.relatedFaceIds; - delete identity.relatedFacePhotoIds; - delete identity.relatedFaceDescriptorIds; - delete identity.relatedIdentityDescriptors; }); - /* If no ID was provided (so no 'filter') then this call is returning + /* If no ID was provided then this call is returning * a list of all identities -- we create a fake identity for all * unlabeled faces */ - if (!filter) { + if (!id) { const unknownIdentity = await getUnknownIdentity(1) identities.push(unknownIdentity); }