ketr.photos/scanner/scanner.c

#include <stdio.h>
#include <stdlib.h>
#include <dirent.h>
#include <sys/types.h>
#include <string.h>
#include <math.h>
#include <sqlite3.h>

typedef struct Face {
  long double descriptor[128];
  int faceId;
  long double *distances;
  struct Face *next;
  struct Face *prev;
} Face;

char fileBuf[5000];
char pathBuf[1028];

Face *readFaceDescriptor(int id, char *path) {
  FILE *f;
  Face *pFace = (Face *)malloc(sizeof(Face));
  memset(pFace, 0, sizeof(Face));
  f = fopen(path, "r");
  if (!f) {
    free(pFace);
    return NULL;
  }
  size_t s = fread(fileBuf, 1, sizeof(fileBuf), f);
  fclose(f);

  char *p = fileBuf;
  fileBuf[s] = 0;
  while (*p && *p != '-' && *p != '+' && (*p < '0' || *p > '9')) {
    p++;
  }
  for (int i = 0; i < 128; i++) {
    char *start = p;
    while (*p && *p != ',' && *p != ']' && *p != ' ' && *p != '\n') {
      p++;
    }
    if (!*p) {
      break;
    }
    *p++ = 0;
    sscanf(start, "%Lf", &pFace->descriptor[i]);
  }

  pFace->faceId = id;
  pFace->next = pFace->prev = NULL;

  return pFace;
}

long double euclideanDistance(long double *a, long double *b) {
  long double sum = 0.0;
  for (int i = 0; i < 128; i++) {
    long double delta = a[i] - b[i];
    sum += delta * delta;
  }
  return sqrtl(sum);
}

int main(int argc, char *argv[]) {
  int maxId = 0;
  int len = 0;
  int i;
  Face *pChain = NULL;
  for (i = 0; i < 100; i++) {
    sprintf(pathBuf, "%s/face-data/%d", argv[1], i);
    DIR *faceDir = opendir(pathBuf);
    fprintf(stderr, "Reading %s...\n", pathBuf);

    if (!faceDir) {
      fprintf(stderr, "Can not open %s\n", pathBuf);
      continue;
    } 

    struct dirent *ent;
    while ((ent = readdir(faceDir)) != NULL) {
      if (strstr(ent->d_name, ".json") == NULL) {
        continue;
      }
      int id = 0;
      char *p = ent->d_name;
      while (*p && *p != '-') {
        id *= 10;
        id += *p - '0';
        p++;
      }
      char path[1028*2];
      sprintf(path, "%s/%s", pathBuf, ent->d_name);
      maxId = maxId > id ? maxId : id;
      Face *pFace = readFaceDescriptor(id, path);
      if (!pFace) {
        continue;
      }
      len++;
      if (len % 1000 == 0) {
        fprintf(stderr, "...read %d...\n", len);
      }
      if (pChain) {
        pFace->next = pChain;
      }
      pChain = pFace;
    }
    closedir(faceDir);
  }

  fprintf(stderr, "Read %d face descriptors...\n", len);

  /* Allocate storage for all distances */
  Face *pLink = pChain;
  while (pLink) {
    pLink->distances = (long double *)malloc(sizeof(long double) * len);
    pLink = pLink->next;
  }

  sqlite3 *db;

  int rc = sqlite3_open("db/photos.db", &db);
  if (rc != SQLITE_OK) {
    fprintf(stderr, "Cannot open database: %s\n", sqlite3_errmsg(db));
    sqlite3_close(db);
    return 1;
  }

  fprintf(stderr, "DB opened.");

  char *err_msg = NULL;
  char *sql = 
    "DELETE FROM facedistances;"
    "BEGIN TRANSACTION;";

  rc = sqlite3_exec(db, sql, 0, 0, &err_msg);
  if (rc != SQLITE_OK ) {
    fprintf(stderr, "SQL error: %s\n", err_msg);
    sqlite3_free(err_msg);        
    sqlite3_close(db);
    return 1;
  } 

  fprintf(stderr, "facedistances deleted and transaction started.\n");

  char sqlBuf[1024];
  pLink = pChain;
  int sourceIndex = 0, lines = 0;
  while (pLink) {
    int targetIndex = 0;
    Face *pTarget = pChain;
    while (pTarget) {
      if (targetIndex == sourceIndex) {
        pLink->distances[targetIndex] = 0.0;
        pTarget->distances[sourceIndex] = 0.0;
      } else {
        if (pLink->distances[targetIndex] == 0.0) {
          pLink->distances[targetIndex] = 
            pTarget->distances[sourceIndex] = euclideanDistance(pLink->descriptor, pTarget->descriptor);
          if (pLink->distances[targetIndex] < 0.5) {
            sprintf(sqlBuf, "INSERT INTO facedistances (face1Id,face2Id,distance) VALUES (%d,%d,%Lf);", 
              ((pLink->faceId < pTarget->faceId) ? pLink->faceId : pTarget->faceId),
              ((pLink->faceId < pTarget->faceId) ? pTarget->faceId : pLink->faceId),
              pLink->distances[targetIndex]);
            rc = sqlite3_exec(db, sqlBuf, 0, 0, &err_msg);
            if (rc != SQLITE_OK ) {
              fprintf(stderr, "SQL error: %s\n", err_msg);
              sqlite3_free(err_msg);        
              sqlite3_close(db);
              return 1;
            } 
            lines++;
            if (lines % 1000 == 0) {
              fprintf(stderr, "...output %d DB lines (%0.2f complete)...\n", lines, (float)(1. * sourceIndex / (1. * len)));
            }
          }
        }
      }
      pTarget = pTarget->next;
      targetIndex++;
    }
    pLink = pLink->next;
    sourceIndex++;
  }

  sprintf(sqlBuf, "UPDATE faces SET lastComparedId=%d;", maxId);

  rc = sqlite3_exec(db, "COMMIT;", 0, 0, &err_msg);
  if (rc != SQLITE_OK ) {
    fprintf(stderr, "SQL error: %s\n", err_msg);
    sqlite3_free(err_msg);        
    sqlite3_close(db);
    return 1;
  } 

  sqlite3_close(db);

  return 0;
}