diff --git a/README.md b/README.md index c59c797..cbe132e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,81 @@ # Ketr Settlers - ```bash sudo apt-get install nodejs npm sudo -E npm install --global npm@latest ``` +# Ketr.Ketran REST API + +## POST /api/v1/game + +### Request + +```json +{} +```` + +### Response + +```json +{ + gameId: id + gameState: { + tiles: [] + + } +} +``` + +# Configuring / installing + + +## Build +```bash +git clone ... +cd server +npm install +``` + +## Install +```bash +sudo cp ketr.ketran /etc/logrotate.d/ +sudo cp ketr.ketran.service /etc/systemd/system/ +sudo systemctl daemon-reload +``` + +Install the following into your nginx server configuration: + +```nginx + location ~ /ketr.ketran/api/.* { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_set_header X-NginX-Proxy true; + proxy_pass_header Set-Cookie; + proxy_pass_header P3P; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_pass http://localhost:8930; + } +``` + +Add security tokens in ketr.ketran/config/local.json: + +```bash +cat << EOF > config/local.json +{ + "tokens": [ { + "$(whoami)": "$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo;)" + } ] +} +EOF +``` + +## Launch +```bash +sudo systemctl start ketr.ketran +``` + diff --git a/ketr.ketran/config/default.json b/config/default.json similarity index 100% rename from ketr.ketran/config/default.json rename to config/default.json diff --git a/config/local.json b/config/local.json new file mode 100644 index 0000000..353fed6 --- /dev/null +++ b/config/local.json @@ -0,0 +1,5 @@ +{ + "tokens": [ { + "jketreno": "1MhGsldnwNkH9d2s-yu8fxZ0JcHCpClY" + } ] +} diff --git a/ketr.ketran/config/production.json b/config/production.json similarity index 100% rename from ketr.ketran/config/production.json rename to config/production.json diff --git a/ketr.ketran/.gitignore b/f/.gitignore similarity index 100% rename from ketr.ketran/.gitignore rename to f/.gitignore diff --git a/ketr.ketran/package.json b/f/package.json similarity index 100% rename from ketr.ketran/package.json rename to f/package.json diff --git a/ketr.ketran/ketr.ketran b/ketr.ketran similarity index 100% rename from ketr.ketran/ketr.ketran rename to ketr.ketran diff --git a/ketr.ketran/ketr.ketran.service b/ketr.ketran.service similarity index 100% rename from ketr.ketran/ketr.ketran.service rename to ketr.ketran.service diff --git a/ketr.ketran/README.md b/ketr.ketran/README.md deleted file mode 100644 index 76d2136..0000000 --- a/ketr.ketran/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Ketr.Ketran REST API - -## POST /api/v1/game - -### Request - -```json -{} -```` - -### Response - -```json -{ - gameId: id - gameState: { - tiles: [] - - } -} -``` - -# Configuring / installing - - -## Build -```bash -git clone ... -cd server -npm install -``` - -## Install -```bash -sudo cp ketr.ketran /etc/logrotate.d/ -sudo cp ketr.ketran.service /etc/systemd/system/ -sudo systemctl daemon-reload -``` - -Install the following into your nginx server configuration: - -```nginx - location ~ /ketr.ketran/api/.* { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - proxy_set_header X-NginX-Proxy true; - proxy_pass_header Set-Cookie; - proxy_pass_header P3P; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://localhost:8930; - } -``` - -Add security tokens in ketr.ketran/config/local.json: - -```bash -cat << EOF > config/local.json -{ - "tokens": [ { - "$(whoami)": "$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo;)" - } ] -} -EOF -``` - -## Launch -```bash -sudo systemctl start ketr.ketran -``` - diff --git a/ketr.ketran/manifest.json b/ketr.ketran/manifest.json deleted file mode 100644 index 9a86cff..0000000 --- a/ketr.ketran/manifest.json +++ /dev/null @@ -1,569 +0,0 @@ -[ - { - "directory": true, - "date_modified": "2020-03-30T18:29:16+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux", - "size": 4096, - "path": "linux" - }, - { - "directory": true, - "date_modified": "2020-03-30T18:29:16+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu", - "size": 4096, - "path": "linux/ubuntu" - }, - { - "directory": true, - "date_modified": "2020-03-30T18:29:23+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10", - "size": 8192, - "path": "linux/ubuntu/19.10" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:15:08+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-cmc-2.0+embargo-169.u19.10-release.x86_64.deb", - "size": 31039028, - "path": "linux/ubuntu/19.10/intel-cmc-2.0+embargo-169.u19.10-release.x86_64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:22:21+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-gpu-tools-dbg_1.24+embargo169_amd64.deb", - "size": 6465620, - "path": "linux/ubuntu/19.10/intel-gpu-tools-dbg_1.24+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:22:21+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-gpu-tools_1.24+embargo169_amd64.deb", - "size": 1955740, - "path": "linux/ubuntu/19.10/intel-gpu-tools_1.24+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:13:12+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-level-zero-gpu_0.8.016262+embargo169_amd64.deb", - "size": 1107520, - "path": "linux/ubuntu/19.10/intel-level-zero-gpu_0.8.016262+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:35:29+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-media-va-driver-non-free_20.1~pre+embargo169_amd64.deb", - "size": 6244784, - "path": "linux/ubuntu/19.10/intel-media-va-driver-non-free_20.1~pre+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:18:25+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-metrics-discovery_1.5.114+embargo169_amd64.deb", - "size": 715152, - "path": "linux/ubuntu/19.10/intel-metrics-discovery_1.5.114+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:19:21+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-metrics-library_1.0.1+embargo169_amd64.deb", - "size": 157448, - "path": "linux/ubuntu/19.10/intel-metrics-library_1.0.1+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:57:35+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-opencl-icd_016320+embargo169_amd64.deb", - "size": 1223400, - "path": "linux/ubuntu/19.10/intel-opencl-icd_016320+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:57:52+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/level-zero-dev_0.91.7+embargo169_amd64.deb", - "size": 90364, - "path": "linux/ubuntu/19.10/level-zero-dev_0.91.7+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:57:52+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/level-zero_0.91.7+embargo169_amd64.deb", - "size": 49700, - "path": "linux/ubuntu/19.10/level-zero_0.91.7+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:36+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-amdgpu1_2.4.100+embargo169_amd64.deb", - "size": 28272, - "path": "linux/ubuntu/19.10/libdrm-amdgpu1_2.4.100+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:36+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-common_2.4.100+embargo169_all.deb", - "size": 14208, - "path": "linux/ubuntu/19.10/libdrm-common_2.4.100+embargo169_all.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:36+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-dev_2.4.100+embargo169_amd64.deb", - "size": 120324, - "path": "linux/ubuntu/19.10/libdrm-dev_2.4.100+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:36+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-intel1_2.4.100+embargo169_amd64.deb", - "size": 70724, - "path": "linux/ubuntu/19.10/libdrm-intel1_2.4.100+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:36+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-nouveau2_2.4.100+embargo169_amd64.deb", - "size": 26316, - "path": "linux/ubuntu/19.10/libdrm-nouveau2_2.4.100+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:36+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-radeon1_2.4.100+embargo169_amd64.deb", - "size": 29580, - "path": "linux/ubuntu/19.10/libdrm-radeon1_2.4.100+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:36+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm2_2.4.100+embargo169_amd64.deb", - "size": 41172, - "path": "linux/ubuntu/19.10/libdrm2_2.4.100+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libegl-mesa0_20.1.0-devel+embargo169_amd64.deb", - "size": 139724, - "path": "linux/ubuntu/19.10/libegl-mesa0_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libegl1-mesa-dev_20.1.0-devel+embargo169_amd64.deb", - "size": 51120, - "path": "linux/ubuntu/19.10/libegl1-mesa-dev_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libegl1-mesa_20.1.0-devel+embargo169_amd64.deb", - "size": 49392, - "path": "linux/ubuntu/19.10/libegl1-mesa_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgbm-dev_20.1.0-devel+embargo169_amd64.deb", - "size": 52940, - "path": "linux/ubuntu/19.10/libgbm-dev_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgbm1_20.1.0-devel+embargo169_amd64.deb", - "size": 71004, - "path": "linux/ubuntu/19.10/libgbm1_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgl1-mesa-dev_20.1.0-devel+embargo169_amd64.deb", - "size": 49408, - "path": "linux/ubuntu/19.10/libgl1-mesa-dev_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgl1-mesa-dri_20.1.0-devel+embargo169_amd64.deb", - "size": 9929788, - "path": "linux/ubuntu/19.10/libgl1-mesa-dri_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgl1-mesa-glx_20.1.0-devel+embargo169_amd64.deb", - "size": 49412, - "path": "linux/ubuntu/19.10/libgl1-mesa-glx_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libglapi-mesa_20.1.0-devel+embargo169_amd64.deb", - "size": 69888, - "path": "linux/ubuntu/19.10/libglapi-mesa_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgles2-mesa-dev_20.1.0-devel+embargo169_amd64.deb", - "size": 49416, - "path": "linux/ubuntu/19.10/libgles2-mesa-dev_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgles2-mesa_20.1.0-devel+embargo169_amd64.deb", - "size": 49400, - "path": "linux/ubuntu/19.10/libgles2-mesa_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libglx-mesa0_20.1.0-devel+embargo169_amd64.deb", - "size": 182580, - "path": "linux/ubuntu/19.10/libglx-mesa0_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:48:07+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigc-dev_8020561+embargo169_amd64.deb", - "size": 2376, - "path": "linux/ubuntu/19.10/libigc-dev_8020561+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:48:07+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigc-tools_8020561+embargo169_amd64.deb", - "size": 1830108, - "path": "linux/ubuntu/19.10/libigc-tools_8020561+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:48:07+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigc1_8020561+embargo169_amd64.deb", - "size": 11786756, - "path": "linux/ubuntu/19.10/libigc1_8020561+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:48:07+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigdfcl-dev_8020561+embargo169_amd64.deb", - "size": 125484, - "path": "linux/ubuntu/19.10/libigdfcl-dev_8020561+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:48:07+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigdfcl1_8020561+embargo169_amd64.deb", - "size": 18982892, - "path": "linux/ubuntu/19.10/libigdfcl1_8020561+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:45+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigdgmm-dev_8020561+embargo169_amd64.deb", - "size": 1126848, - "path": "linux/ubuntu/19.10/libigdgmm-dev_8020561+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:45+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigdgmm6_8020561+embargo169_amd64.deb", - "size": 127364, - "path": "linux/ubuntu/19.10/libigdgmm6_8020561+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:35:29+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigfxcmrt-dev_20.1~pre+embargo169_amd64.deb", - "size": 69096, - "path": "linux/ubuntu/19.10/libigfxcmrt-dev_20.1~pre+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:35:29+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigfxcmrt7_20.1~pre+embargo169_amd64.deb", - "size": 30668, - "path": "linux/ubuntu/19.10/libigfxcmrt7_20.1~pre+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:17:58+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libmfx-dev_20.1~pre+embargo169_amd64.deb", - "size": 39952, - "path": "linux/ubuntu/19.10/libmfx-dev_20.1~pre+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:17:58+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libmfx-tools_20.1~pre+embargo169_amd64.deb", - "size": 1473148, - "path": "linux/ubuntu/19.10/libmfx-tools_20.1~pre+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:17:58+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libmfx1_20.1~pre+embargo169_amd64.deb", - "size": 2833028, - "path": "linux/ubuntu/19.10/libmfx1_20.1~pre+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libosmesa6-dev_20.1.0-devel+embargo169_amd64.deb", - "size": 52776, - "path": "linux/ubuntu/19.10/libosmesa6-dev_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libosmesa6_20.1.0-devel+embargo169_amd64.deb", - "size": 2741820, - "path": "linux/ubuntu/19.10/libosmesa6_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:23:06+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva-dev_2.7.0.1+embargo169_amd64.deb", - "size": 104500, - "path": "linux/ubuntu/19.10/libva-dev_2.7.0.1+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:23:06+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva-drm2_2.7.0.1+embargo169_amd64.deb", - "size": 17760, - "path": "linux/ubuntu/19.10/libva-drm2_2.7.0.1+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:23:06+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva-glx2_2.7.0.1+embargo169_amd64.deb", - "size": 21344, - "path": "linux/ubuntu/19.10/libva-glx2_2.7.0.1+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:23:06+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva-wayland2_2.7.0.1+embargo169_amd64.deb", - "size": 19664, - "path": "linux/ubuntu/19.10/libva-wayland2_2.7.0.1+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:23:06+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva-x11-2_2.7.0.1+embargo169_amd64.deb", - "size": 22392, - "path": "linux/ubuntu/19.10/libva-x11-2_2.7.0.1+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:23:06+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva2_2.7.0.1+embargo169_amd64.deb", - "size": 60736, - "path": "linux/ubuntu/19.10/libva2_2.7.0.1+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libwayland-egl1-mesa_20.1.0-devel+embargo169_amd64.deb", - "size": 49416, - "path": "linux/ubuntu/19.10/libwayland-egl1-mesa_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libxatracker-dev_20.1.0-devel+embargo169_amd64.deb", - "size": 52896, - "path": "linux/ubuntu/19.10/libxatracker-dev_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libxatracker2_20.1.0-devel+embargo169_amd64.deb", - "size": 1644200, - "path": "linux/ubuntu/19.10/libxatracker2_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:20+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/linux-headers-5.4.17-169+_5.4.17-169+-1_amd64.deb", - "size": 11420780, - "path": "linux/ubuntu/19.10/linux-headers-5.4.17-169+_5.4.17-169+-1_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T16:55:17+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/linux-i915-firmware_2020.12+embargo169.deb", - "size": 4758832, - "path": "linux/ubuntu/19.10/linux-i915-firmware_2020.12+embargo169.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:21+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/linux-image-5.4.17-169+-dbg_5.4.17-169+-1_amd64.deb", - "size": 903956216, - "path": "linux/ubuntu/19.10/linux-image-5.4.17-169+-dbg_5.4.17-169+-1_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:21+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/linux-image-5.4.17-169+_5.4.17-169+-1_amd64.deb", - "size": 60596400, - "path": "linux/ubuntu/19.10/linux-image-5.4.17-169+_5.4.17-169+-1_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:20:21+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/linux-libc-dev_5.4.17-169+-1_amd64.deb", - "size": 1069796, - "path": "linux/ubuntu/19.10/linux-libc-dev_5.4.17-169+-1_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T16:55:07+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/manifest.yml", - "size": 5595, - "path": "linux/ubuntu/19.10/manifest.yml" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/mesa-common-dev_20.1.0-devel+embargo169_amd64.deb", - "size": 677236, - "path": "linux/ubuntu/19.10/mesa-common-dev_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/mesa-opencl-icd_20.1.0-devel+embargo169_amd64.deb", - "size": 10299076, - "path": "linux/ubuntu/19.10/mesa-opencl-icd_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/mesa-va-drivers_20.1.0-devel+embargo169_amd64.deb", - "size": 2684040, - "path": "linux/ubuntu/19.10/mesa-va-drivers_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/mesa-vdpau-drivers_20.1.0-devel+embargo169_amd64.deb", - "size": 2810196, - "path": "linux/ubuntu/19.10/mesa-vdpau-drivers_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:27:11+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/mesa-vulkan-drivers_20.1.0-devel+embargo169_amd64.deb", - "size": 3587736, - "path": "linux/ubuntu/19.10/mesa-vulkan-drivers_20.1.0-devel+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:23:06+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/va-driver-all_2.7.0.1+embargo169_amd64.deb", - "size": 13948, - "path": "linux/ubuntu/19.10/va-driver-all_2.7.0.1+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T17:23:06+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/vainfo_2.6.0.0+embargo169_amd64.deb", - "size": 18908, - "path": "linux/ubuntu/19.10/vainfo_2.6.0.0+embargo169_amd64.deb" - }, - { - "directory": true, - "date_modified": "2020-03-30T18:29:26+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils", - "size": 4096, - "path": "utils" - }, - { - "directory": true, - "date_modified": "2020-03-30T18:29:26+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu", - "size": 4096, - "path": "utils/ubuntu" - }, - { - "directory": true, - "date_modified": "2020-03-30T18:29:27+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10", - "size": 4096, - "path": "utils/ubuntu/19.10" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:29:15+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/crucible_1.0.20200316+i169_amd64.deb", - "size": 14641300, - "path": "utils/ubuntu/19.10/crucible_1.0.20200316+i169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:20:14+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/libwaffle-1-0_1.6.90+embargo169_amd64.deb", - "size": 28100, - "path": "utils/ubuntu/19.10/libwaffle-1-0_1.6.90+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:20:14+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/libwaffle-dev_1.6.90+embargo169_amd64.deb", - "size": 7444, - "path": "utils/ubuntu/19.10/libwaffle-dev_1.6.90+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:20:14+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/libwaffle-doc_1.6.90+embargo169_all.deb", - "size": 4504, - "path": "utils/ubuntu/19.10/libwaffle-doc_1.6.90+embargo169_all.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:29:15+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/piglit-shards_20200330+embargo169.deb", - "size": 15768, - "path": "utils/ubuntu/19.10/piglit-shards_20200330+embargo169.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:28:06+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/piglit_1.0.20200212+embargo169_amd64.deb", - "size": 33672144, - "path": "utils/ubuntu/19.10/piglit_1.0.20200212+embargo169_amd64.deb" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:20:14+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/waffle-utils_1.6.90+embargo169_amd64.deb", - "size": 11236, - "path": "utils/ubuntu/19.10/waffle-utils_1.6.90+embargo169_amd64.deb" - }, - { - "directory": true, - "date_modified": "2020-03-30T18:29:27+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/TestTools", - "size": 4096, - "path": "TestTools" - }, - { - "directory": true, - "date_modified": "2020-03-30T18:29:27+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/TestTools/Linux", - "size": 4096, - "path": "TestTools/Linux" - }, - { - "directory": false, - "date_modified": "2020-03-30T18:29:16+0000", - "url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/TestTools/Linux/TestTools-Release-Internal-x64.tar.xz", - "size": 368, - "path": "TestTools/Linux/TestTools-Release-Internal-x64.tar.xz" - } -] \ No newline at end of file diff --git a/ketr.ketran/server/face-recognizer.js b/ketr.ketran/server/face-recognizer.js deleted file mode 100644 index da8fcc3..0000000 --- a/ketr.ketran/server/face-recognizer.js +++ /dev/null @@ -1,404 +0,0 @@ -/* -* Face recognition: -* 1. For each photo, extract all faces. Store face rectangles. -* face_id unique -* photo_id foreign key -* top left bottom right -* identity_id -* distance (0 == truth; manually assigned identity) -* 2. For each face_id, create: -* /${picturesPath}face-data/${face_id % 100}/ -* ${face_id}-normalized -* ${face_id}-original -* ${face_id}-data -*/ - -"use strict"; - -process.env.TZ = "Etc/GMT"; - -console.log("Loading face-recognizer"); - -require('@tensorflow/tfjs-node'); - -const config = require("config"), - Promise = require("bluebird"), - { exists, mkdir, unlink } = require("./lib/util"), - faceapi = require("face-api.js"), - fs = require("fs"), - canvas = require("canvas"); - -const { createCanvas, Canvas, Image, ImageData } = canvas; - -faceapi.env.monkeyPatch({ Canvas, Image, ImageData }); - -const maxConcurrency = require("os").cpus().length; - -require("./console-line.js"); /* Monkey-patch console.log with line numbers */ - -const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/", - faceData = picturesPath + "face-data/"; - -let photoDB = null; - -console.log("Loading pictures out of: " + picturesPath); - -function alignFromLandmarks(image, landmarks) { - const faceMargin = 0.3, - width = 256, height = 256, - dY = landmarks._positions[45]._y - landmarks._positions[36]._y, - dX = landmarks._positions[45]._x - landmarks._positions[36]._x, - mid = { - x: landmarks._positions[36]._x + 0.5 * dX, - y: landmarks._positions[36]._y + 0.5 * dY - }, - rotation = -Math.atan2(dY, dX), - cosRotation = Math.cos(rotation), - sinRotation = Math.sin(rotation), - eyeDistance = Math.sqrt(dY * dY + dX * dX), - scale = width * (1.0 - 2. * faceMargin) / eyeDistance, - canvas = createCanvas(width, height), - ctx = canvas.getContext("2d"); - - const prime = { - x: mid.x * cosRotation - mid.y * sinRotation, - y: mid.y * cosRotation + mid.x * sinRotation - }; - - mid.x = prime.x; - mid.y = prime.y; - - ctx.translate( - 0.5 * width - mid.x * scale, - 0.5 * height - (height * (0.5 - faceMargin)) - mid.y * scale); - ctx.rotate(rotation); - ctx.scale(scale, scale); - ctx.drawImage(image, 0, 0); - /* - ctx.strokeStyle = "red"; - ctx.strokeWidth = "1"; - ctx.beginPath(); - landmarks._positions.forEach((point, index) => { - if (index == 0) { - ctx.moveTo(point._x, point._y); - } else { - ctx.lineTo(point._x, point._y); - } - }); - ctx.stroke(); - */ - return canvas; -} - -process.stdout.write("Loading DB."); -require("./db/photos").then(function(db) { - process.stdout.write("done\n"); - photoDB = db; -}).then(() => { - console.log("DB connected."); - process.stdout.write("Loading models."); - return faceapi.nets.ssdMobilenetv1.loadFromDisk('./models'); -}).then(() => { - process.stdout.write("."); - return faceapi.nets.faceLandmark68Net.loadFromDisk('./models'); -}).then(() => { - process.stdout.write("."); - return faceapi.nets.faceRecognitionNet.loadFromDisk('./models'); -}).then(() => { - console.log("Beginning face detection scanning."); - return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " + - "FROM photos " + - "LEFT JOIN albums ON (albums.id=photos.albumId) " + - "WHERE faces=-1 AND photos.duplicate=0 AND photos.deleted=0 ORDER BY albums.path,photos.filename", { - type: photoDB.sequelize.QueryTypes.SELECT, - raw: true - }); -}).then((needToScan) => { - const total = needToScan.length; - let remaining = total, - processed = 0, - lastStatus = Date.now(); - - console.log(`${needToScan.length} photos have not had faces scanned.`); - - return Promise.map(needToScan, (photo) => { - const photoPath = photo.path + photo.filename; - - console.log(`Processing ${photoPath}...`); - - /* Remove any existing face data for this photo */ - return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", { - replacements: photo, - type: photoDB.sequelize.QueryTypes.SELECT, - raw: true - }).then((faces) => { - /* For each face-id, remove any face-data files, and then remove all the entries - * from the DB */ - return Promise.map(faces, (face) => { - return Promise.map([ "-data.json", "-original.png" ], (suffix) => { - const id = face.id, - dataPath = faceData + (id % 100) + "/" + id + suffix; - return exists(dataPath).then((result) => { - if (result) { - console.log(`...removing ${dataPath}`); - return unlink(dataPath); - } - }); - }); - }).then(() => { - return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", { - replacements: photo, - }); - }); - }).then(async () => { - /* Process image for faces data */ - const image = await canvas.loadImage(picturesPath + photoPath); - const detections = await faceapi.detectAllFaces(image, - new faceapi.SsdMobilenetv1Options({ - minConfidence: 0.9 - }) - ).withFaceLandmarks(); - - if (detections.length > 0) { - console.log(`...${detections.length} faces identified in ${photoPath}.`); - } - - return Promise.map(detections, async (face) => { - const detection = face.detection, - canvas = alignFromLandmarks(image, face.landmarks); - face.descriptor = await faceapi.computeFaceDescriptor(canvas); - - const width = detection._box._width, - height = detection._box._height, - replacements = { - id: photo.id, - top: detection._box._y / detection._imageDims.height, - left: detection._box._x / detection._imageDims.width, - bottom: (detection._box._y + height) / detection._imageDims.height, - right: (detection._box._x + width) / detection._imageDims.width, - faceConfidence: detection._score - }; - - return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " + - "VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", { - replacements: replacements - }).spread((results, metadata) => { - return metadata.lastID; - }).then((id) => { - const path = faceData + (id % 100); - return mkdir(path).then(() => { - const dataPath = `${path}/${id}-data.json`, data = []; - console.log(`...writing descriptor data to ${dataPath}...`); - /* Confert from sparse object to dense array */ - for (let i = 0; i < 128; i++) { - data.push(face.descriptor[i]); - } - fs.writeFileSync(dataPath, JSON.stringify(data)); - }).then(() => { - const target = `${path}/${id}-original.png`; - console.log(`...writing aligned face crop to ${target}.`); - fs.writeFileSync(target, canvas.toBuffer("image/png", { - quality: 0.95, - chromaSubsampling: false - })); - }).catch((error) => { - console.error(error); - process.exit(-1); - }); - }); - }).then(() => { - return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", { - replacements: { - id: photo.id, - faces: detections.length - }, - }); - }); - }).catch((error) => { - console.log(error); - console.warn("Skipping out on image " + photoPath + " and marking to 0 faces to prevent future scanning."); - return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", { - replacements: { - id: photo.id, - faces: 0 - }, - }); - }).then(() => { - processed++; - const now = Date.now(); - if (now - lastStatus > 5000) { - const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10, - eta = Math.round((total - processed) / rate); - lastStatus = now; - remaining = total - processed; - console.log(`Processing ${rate} images per second. ${remaining} images to be processed. ETA: ${eta}s`); - } - }); - }, { - concurrency: maxConcurrency - }); -}).then(() => { - console.log("Looking for face distances that need to be updated..."); - let maxId; - - return photoDB.sequelize.query("SELECT faces.id FROM faces ORDER BY faces.id DESC LIMIT 1", { - type: photoDB.sequelize.QueryTypes.SELECT, - raw: true - }).then((results) => { - if (!results.length) { - console.log("...no faces exist yet to generate distances."); - maxId = 0; - return []; - } - maxId = results[0].id; - return photoDB.sequelize.query( - "SELECT faces.id,faces.lastComparedId " + - "FROM faces INNER JOIN photos ON photos.duplicate=0 AND photos.deleted=0 AND photos.id=faces.photoId " + - "WHERE faces.lastComparedId<:maxId OR faces.lastComparedId IS NULL " + - "ORDER BY faces.id ASC", { - replacements: { - maxId: maxId - }, - type: photoDB.sequelize.QueryTypes.SELECT, - raw: true - }); - }).then((facesToUpdate) => { - console.log(`...${facesToUpdate.length} faces need distances updated.`); - console.log("---- run scanner/scanner !! ---"); - return []; - if (facesToUpdate.length == 0) { - return facesToUpdate; - } - - const descriptors = {}; - - return photoDB.sequelize.query( - "SELECT id FROM faces ORDER BY id ASC", { - type: photoDB.sequelize.QueryTypes.SELECT, - raw: true - }).then((allFaces) => { - console.log(`...reading ${allFaces.length} descriptors...`); - return Promise.map(allFaces, (face) => { - const id = face.id, - dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json"; - - if (id in descriptors) { - return; - } - - return exists(dataPath).then((doesExist) => { - if (!doesExist) { - console.warn(`${dataPath} is missing!`); - return; - } - - descriptors[id] = JSON.parse(fs.readFileSync(dataPath)); - }); - }); - }).then(() => { - const total = facesToUpdate.length; - let remaining = total, - processed = 0, - lastStatus = Date.now(), - targets = []; - - for (let target in descriptors) { - targets.push({ id: target, descriptor: descriptors[target] }); - } - - return Promise.mapSeries(facesToUpdate, (face) => { - if (!(face.id in descriptors)) { - console.warn(`...attempt to compare distance with no descriptor for ${face.id}`); - return; - } - - const faceDescriptor = descriptors[face.id]; - - return photoDB.sequelize.transaction((transaction) => { - return photoDB.sequelize.query( - "SELECT distance,face1Id,face2Id " + - "FROM facedistances " + - "WHERE face1Id=:id OR face2Id=:id " + - "ORDER BY face1Id ASC", { - replacements: { - id: face.id - }, - type: photoDB.sequelize.QueryTypes.SELECT, - raw: true, - transaction: transaction - }).then((distances) => { - return Promise.map(targets, (target) => { - /* Skip comparing to self */ - if (target.id == face.id) { - return; - } - - /* Only compare against newer faces */ - if (face.lastComparedId && target.id <= face.lastComparedId) { - return; - } - - const index = distances.findIndex((distance) => { - return distance.face1Id == target.id || distance.face2Id == target.id - }); - - if (index != -1) { - /* A distance has already been calculated between face and target */ - return; - } - - const distance = faceapi.euclideanDistance(faceDescriptor, target.descriptor); - - /* If the distance > 0.6, we don't want to store this in the DB */ - if (distance > 0.6) { - return; - } - - if (distance < 0.4) { - process.stdout.write("."); -// console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`); - } - - return photoDB.sequelize.query( - "INSERT INTO facedistances (face1Id,face2Id,distance) " + - "VALUES (:first,:second,:distance)", { - replacements: { - first: Math.min(face.id, target.id), - second: Math.max(face.id, target.id), - distance: distance - }, - transaction: transaction - }); - }, { - concurrency: maxConcurrency - }); - }).then(() => { - return photoDB.sequelize.query( - "UPDATE faces SET lastComparedId=:lastId WHERE id=:id", { - replacements: { - lastId: maxId, - id: face.id - }, - transaction: transaction - }); - }); - }).then(() => { - processed++; - const now = Date.now(); - if (now - lastStatus > 5000) { - const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10, - eta = Math.round((total - processed) / rate); - lastStatus = now; - remaining = total - processed; - console.log(`\nProcessing ${rate} faces per second. ${remaining} faces to be processed. ETA: ${eta}s`); - } - }); - }); - }); - }); -}).then(() => { - console.log("Face detection scanning completed."); -}).catch((error) => { - console.error(error); - process.exit(-1); -}); diff --git a/ketr.ketran/server/face.js b/ketr.ketran/server/face.js deleted file mode 100644 index eccce41..0000000 --- a/ketr.ketran/server/face.js +++ /dev/null @@ -1,281 +0,0 @@ -"use strict"; - -process.env.TZ = "Etc/GMT"; - -require('@tensorflow/tfjs-node'); - -let photoDB = null; - -const config = require("config"), - Promise = require("bluebird"), - { exists, mkdir, unlink } = require("./lib/util"), - faceapi = require("face-api.js"), - fs = require("fs"), - canvas = require("canvas"); - -const { createCanvas, Canvas, Image, ImageData } = canvas; - -faceapi.env.monkeyPatch({ Canvas, Image, ImageData }); - -const maxConcurrency = require("os").cpus().length; - -require("./console-line.js"); /* Monkey-patch console.log with line numbers */ - -const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/", - faceData = picturesPath + "face-data/"; - -function alignFromLandmarks(image, landmarks, drawLandmarks) { - const faceMargin = 0.45, - width = 512, height = 512, - dY = landmarks._positions[45]._y - landmarks._positions[36]._y, - dX = landmarks._positions[45]._x - landmarks._positions[36]._x, - mid = { - x: landmarks._positions[36]._x + 0.5 * dX, - y: landmarks._positions[36]._y + 0.5 * dY - }, - rotation = -Math.atan2(dY, dX), - cosRotation = Math.cos(rotation), - sinRotation = Math.sin(rotation), - eyeDistance = Math.sqrt(dY * dY + dX * dX), - scale = width * (1.0 - 2. * faceMargin) / eyeDistance, - canvas = createCanvas(width, height), - ctx = canvas.getContext("2d"); - - const prime = { - x: mid.x * cosRotation - mid.y * sinRotation, - y: mid.y * cosRotation + mid.x * sinRotation - }; - - mid.x = prime.x; - mid.y = prime.y; - - ctx.translate( - 0.5 * width - mid.x * scale, - 0.5 * height - (height * (0.5 - faceMargin)) - mid.y * scale); - ctx.rotate(rotation); - ctx.scale(scale, scale); - ctx.drawImage(image, 0, 0); - - if (drawLandmarks) { - ctx.strokeStyle = "red"; - ctx.strokeWidth = "1"; - ctx.beginPath(); - landmarks._positions.forEach((point, index) => { - if (index == 0) { - ctx.moveTo(point._x, point._y); - } else { - ctx.lineTo(point._x, point._y); - } - }); - ctx.stroke(); - } - - return canvas; -} - -process.stdout.write("Loading DB."); -require("./db/photos").then(function(db) { - process.stdout.write("done\n"); - photoDB = db; -}).then(() => { - console.log("DB connected."); - process.stdout.write("Loading models."); - return faceapi.nets.ssdMobilenetv1.loadFromDisk('./models'); -}).then(() => { - process.stdout.write("."); - return faceapi.nets.faceLandmark68Net.loadFromDisk('./models'); -}).then(() => { - process.stdout.write("."); - return faceapi.nets.faceRecognitionNet.loadFromDisk('./models'); -}).then(async () => { - process.stdout.write(".done\n"); - - if (process.argv[0].match(/node/)) { - process.argv.shift(); /* node */ - } - process.argv.shift(); /* script name */ - - return Promise.resolve().then(() => { - if (process.argv.length != 0) { - return process.argv; - } - - /* If no parameters provided, scan all faces to create image crops */ - return photoDB.sequelize.query("SELECT id FROM faces ORDER BY id ASC", { - type: photoDB.sequelize.QueryTypes.SELECT, - raw: true - }).then((results) => { - return results.map(result => result.id); - }); - }); -}).then((args) => { - const faces = []; - - console.log(`Scanning ${args.length} faces.`); - return Promise.map(args, (arg) => { - const file = arg; - let id = parseInt(arg); - - let loader; - - if (id == file) { - /* This is a face id */ - console.log(`Looking up face-id ${id}...`); - loader = photoDB.sequelize.query( - "SELECT albums.path,photos.filename,photos.width,photos.height,faces.* " + - "FROM faces,photos,albums " + - "WHERE photos.id=faces.photoId " + - "AND albums.id=photos.albumId " + - "AND faces.id=:id", { - replacements: { - id: id - }, - type: photoDB.sequelize.QueryTypes.SELECT, - raw: true - }).then((results) => { - if (results.length != 1) { - console.error(`...error. No face-id found: ${id}.\n`); - process.exit(-1); - } - const photo = results[0]; - console.log(`...loading ${photo.filename}`); - - const file = photo.path + photo.filename; - return canvas.loadImage(picturesPath + file).then(async (image) => { - const detectors = [ { - detection: { - _box: { - _x: photo.left * photo.width, - _y: photo.top * photo.height, - _width: (photo.right - photo.left) * photo.width, - _height: (photo.bottom - photo.top) * photo.height, - } - }, - descriptor: JSON.parse(fs.readFileSync(faceData + (id % 100) + "/" + id + "-data.json")) - } ]; - return [ file, image, detectors ]; - }); - }); - } else { - /* This is a file */ - console.log(`Loading ${file}...`); - id = undefined; - loader = canvas.loadImage(picturesPath + file).then(async (image) => { - const detectors = await faceapi.detectAllFaces(image, - new faceapi.SsdMobilenetv1Options({ - minConfidence: 0.9 - }) - ).withFaceLandmarks(); - - await detectors.forEach(async (detector, index) => { - const canvas = alignFromLandmarks(image, detector.landmarks, false); - fs.writeFileSync(`rotation-pre-${index}.png`, canvas.toBuffer("image/png", { - quality: 0.95, - chromaSubsampling: false - })); - const detected = await faceapi.detectSingleFace(canvas, - new faceapi.SsdMobilenetv1Options({ - minConfidence: 0.1 - }) - ).withFaceLandmarks(); - const descriptor = await faceapi.computeFaceDescriptor(canvas); - console.log(`Processing face ${index}...`); - console.log(`...pre aligned score: ${detector.detection._score}`); - if (!detected) { - console.log("No face found in re-scaled and aligned image"); - return; - } - console.log(`...post-aligned score: ${detected.detection._score}`); - const newCanvas = alignFromLandmarks(canvas, detected.landmarks, true); - - fs.writeFileSync(`rotation-post-${index}.png`, newCanvas.toBuffer("image/png", { - quality: 0.95, - chromaSubsampling: false - })); - - console.log(`Wrote rotation-${index}.png`); - - const data = []; - /* Confert from sparse object to dense array */ - for (let i = 0; i < 128; i++) { - data.push(descriptor[i]); - } - detector.descriptor = data; - }); - - return [ file, image, detectors ]; - }); - } - - return loader.then((results) => { - const filepath = results[0], - image = results[1], - detectors = results[2]; - - process.stdout.write(`${detectors.length} faces.\n`); - - return Promise.map(detectors, (face, index) => { - faces.push({ - filepath: filepath, - index: index, - descriptor: face.descriptor - }) - - /* If this is a face-id, output the -original.png - * meta-data file */ - if (!id) { - return; - } - - const path = "face-data/" + (id % 100), - target = `${path}/${id}-original.png`, - box = face.detection._box, - aspect = box._width / box._height, - dx = (aspect > 1.0) ? 200 : (200 * aspect), - dy = (aspect < 1.0) ? 200 : (200 / aspect); - - return exists(target).then((doesExist) => { - if (doesExist) { - console.log(`...${target} already exists.`); - return; - } - const canvas = createCanvas(200, 200), - ctx = canvas.getContext('2d'); - - ctx.fillStyle = "rgba(0, 0, 0, 0)"; - ctx.fillRect(0, 0, 200, 200); - ctx.drawImage(image, box._x, box._y, box._width, box._height, - Math.floor((200 - dx) * 0.5), - Math.floor((200 - dy) * 0.5), dx, dy); - - console.log(`...writing to ${target}.`); - - return mkdir(path).then(() => { - fs.writeFileSync(picturesPath + target, canvas.toBuffer("image/png", { - quality: 0.95, - chromaSubsampling: false - })); - }); - }); - }); - }); - }, { - concurrency: maxConcurrency - }).then(() => { - console.log("Face detection scanning completed."); - if (0) faces.forEach((a, i) => { - faces.forEach((b, j) => { - if (i == j) { - return; - } - const distance = faceapi.euclideanDistance(a.descriptor, b.descriptor); - if (distance < 0.4) { - console.log(`${a.filepath}.${a.index} is similar to ${b.filepath}.${b.index}: ${distance}`); - } - }) - }); - }); -}).catch((error) => { - console.error(error); - process.exit(-1); -}); diff --git a/ketr.ketran/server/lib/util.js b/ketr.ketran/server/lib/util.js deleted file mode 100644 index dcbb545..0000000 --- a/ketr.ketran/server/lib/util.js +++ /dev/null @@ -1,88 +0,0 @@ -"use strict"; - -const config = require("config"), - fs = require("fs"), - Promise = require("bluebird"), - picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/"; - -const stat = function (_path) { - if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) { - _path = _path.substring(picturesPath.length); - } - - let path = picturesPath + _path; - - return new Promise(function (resolve, reject) { - fs.stat(path, function (error, stats) { - if (error) { - return reject(error); - } - return resolve(stats); - }); - }); -} - -const unlink = function (_path) { - if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) { - _path = _path.substring(picturesPath.length); - } - - let path = picturesPath + _path; - - return new Promise(function (resolve, reject) { - fs.unlink(path, function (error) { - if (error) { - return reject(error); - } - return resolve(); - }); - }); -} - -const mkdir = function (_path) { - if (_path.indexOf(picturesPath) == 0) { - _path = _path.substring(picturesPath.length); - } - - let parts = _path.split("/"), path; - - parts.unshift(picturesPath); - return Promise.mapSeries(parts, function (part) { - if (!path) { - path = picturesPath.replace(/\/$/, ""); - } else { - path += "/" + part; - } - - return stat(path).catch(function (error) { - if (error.code != "ENOENT") { - throw error; - } - - return new Promise(function (resolve, reject) { - fs.mkdir(path, function (error) { - if (error) { - return reject(error); - } - - return resolve(); - }); - }); - }); - }); -} - -const exists = function(path) { - return stat(path).then(function() { - return true; - }).catch(function() { - return false; - }); -} - -module.exports = { - stat, - exists, - mkdir, - unlink -}; \ No newline at end of file diff --git a/ketr.ketran/server/scanner.js b/ketr.ketran/server/scanner.js deleted file mode 100644 index 3687872..0000000 --- a/ketr.ketran/server/scanner.js +++ /dev/null @@ -1,748 +0,0 @@ -/** - * scanner - * - * Face recognition: - * 1. For each photo, extract all faces. Store face rectangles. - * face_id unique - * photo_id foreign key - * top left bottom right - * identity_id - * distance (0 == truth; manually assigned identity) - * 2. For each face_id, create: - * normalized_file - * original_file - * 128 float - */ -"use strict"; - -/* meta directories are not scanned for photos */ -const metaDirectories = [ "thumbs", "raw", "face-data", ".git", "corrupt" ]; - -const Promise = require("bluebird"), - fs = require("fs"), - config = require("config"), - moment = require("moment"), - crypto = require("crypto"), - { stat, mkdir, exists } = require("./lib/util"); - -let photoDB = null; - -const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/"; - -let processQueue = [], triedClean = [], lastScan = new Date("1800-01-01"); - -//const rawExtension = /\.(nef|orf)$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef", "orf" ]; - -const rawExtension = /\.nef$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef" ]; - -function removeNewerFile(path, fileA, fileB) { - fs.stat(path + fileA, function(err, statsA) { - if (err) { - return; - } - fs.stat(path + fileB, function(err, statsB) { - if (err) { - return; - } - if (statsA.mtime > statsB.mtime) { - setStatus("Removing file by moving to 'corrupt':" + fileA); - moveCorrupt(path, fileA); - } else { - setStatus("Removing file by moving to 'corrupt':" + fileB); - moveCorrupt(path, fileB); - } - }); - }); -} - -let processRunning = false; - -const { spawn } = require('child_process'); - -const sharp = require("sharp"), exif = require("exif-reader"); - -function convertRawToJpg(path, raw, file) { - setStatus(`Converting ${path}${raw} to ${file}.`); - - path = picturesPath + path; - - return new Promise(function(resolve, reject) { - return exists(path + file.replace(rawExtension, ".jpg")).then(function(exist) { - if (exist) { - setStatus("Skipping already converted file: " + file); - return; - } - - const darktable = spawn("darktable-cli", [ - path + raw, - path + file - ]); - - const stderr = []; - darktable.stderr.on('data', function(data) { - stderr.push(data); - }); - - return new Promise((resolve, reject) => { - darktable.on('exit', (code, signal) => { - if (signal || code != 0) { - let error = "darktable for " + path + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n"; - setStatus(error, "error"); - return moveCorrupt(path, file).then(function() { - setStatus("darktable failed", "warn"); - return reject(error); - }).catch(function(error) { - setStatus("moveCorrupt failed", "warn"); - return reject(error); - }); - } - return mkdir(path + "raw").then(function() { - fs.rename(path + raw, path + "raw/" + raw, function(err) { - if (err) { - setStatus("Unable to move RAW file: " + path + raw, "error"); - return reject(err); - } - return resolve(); - }); - }).catch(function(error) { - setStatus("mkdir failed", "warn"); - return reject(error); - }); - }); - }); - }); - }); -} - -function moveCorrupt(path, file) { - if (path.indexOf(picturesPath) != 0) { - path = picturesPath + path; - } - - setStatus("Moving corrupt file '" + file + "' to " + path + "corrupt", "warn"); - - return mkdir(path + "corrupt").then(function() { - return new Promise(function(resolve, reject) { - fs.rename(path + file, path + "corrupt/" + file, function(err) { - if (err) { - setStatus("Unable to move corrupt file: " + path + file, "error"); - return reject(err); - } - return resolve(); - }); - }); - }); -} - -function processBlock(items) { - - if (items) { - processQueue = processQueue.concat(items); - } - - if (processRunning) { - /* Invoke once per second to check if there are items to process */ - setTimeout(processBlock, 1000); - return; - } - - let processing = processQueue.splice(0), needsProcessing = [], duplicates = []; - - processRunning = true; - - /* Sort to newest files to be processed first */ - processing.sort(function(a, b) { - return b.stats.mtime - a.stats.mtime; - }); - - let toProcess = processing.length, lastMessage = moment(); - setStatus("Items to be processed: " + toProcess); - return Promise.mapSeries(processing, (asset) => { - if (!asset.raw) { - return; - } - - const path = asset.album.path; - - return exists(picturesPath + path + asset.filename).then(function(exist) { - if (exist) { - return asset; - } - - return mkdir(picturesPath + path + "raw").then(function() { - return convertRawToJpg(path, asset.raw, asset.filename); - }).then(function() { - console.log("Done converting..."); - }); - }); - }).then(() => { - return Promise.mapSeries(processing, (asset) => { - return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) { - asset.hash = hash; - return asset; - }).then(function(asset) { - return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " + - "LEFT JOIN photos ON (photos.id=photohashes.photoId) " + - "LEFT JOIN albums ON (albums.id=photos.albumId) " + - "WHERE hash=:hash OR photoId=:id", { - replacements: asset, - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(results) { - let query; - - if (results.length == 0) { - query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)"; - } else if (results[0].hash != asset.hash) { - query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id"; - } else if (results[0].photoId != asset.id) { - setStatus("Duplicate asset: " + - "'" + asset.album.path + asset.filename + "' is a copy of " + - "'" + results[0].path + results[0].filename + "'"); - if (asset.duplicate != results[0].photoId) { - asset.duplicate = results[0].photoId; - duplicates.push(asset); - } - return null; - } - - /* Even if the hash doesn't need to be updated, the entry needs to be scanned */ - // console.log("process needed because of " + query); - needsProcessing.push(asset); - - if (!query) { - return asset; - } - - return photoDB.sequelize.query(query, { - replacements: asset, - }).then(function() { - return asset; - }); - }); - }).then(function(asset) { - if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */ - return; - } - - var path = asset.album.path, - file = asset.filename, - created = asset.stats.mtime, - albumId = asset.album.id; - - var src = picturesPath + path + file, - image = sharp(src); - - return image.limitInputPixels(1073741824).metadata().then(function(metadata) { - if (metadata.exif) { - metadata.exif = exif(metadata.exif); - delete metadata.exif.thumbnail; - delete metadata.exif.image; - for (var key in metadata.exif.exif) { - if (Buffer.isBuffer(metadata.exif.exif[key])) { - metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]"; - } - } - } - - asset.width = metadata.width; - asset.height = metadata.height; - asset.added = moment().format(); - - if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) { - asset.taken = moment(metadata.exif.exif.DateTimeOriginal).format(); - asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format(); - - if (asset.taken == "Invalid date" || asset.taken.replace(/T.*/, "") == "1899-11-30") { - setStatus("Invalid EXIF date information for " + asset.album.path + asset.filename); - asset.taken = asset.modified = moment(created).format(); - } - } else { - /* Attempt to infer the datestamp from the filename */ - let date = moment(created).format(); - - let match = file.match(/WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/); - if (match) { - date = moment((match[1]+" "+match[2]), "YYYY-MM-DD h.mm.ss a").format(); - if (date == "Invalid date") { - date = moment(created).format(); - } - } else { - match = file.match(/(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/); - if (match) { - if (match[2]) { /* Stamp had time in it */ - date = moment((match[1]+""+match[2]).replace(/-/g, ""), "YYYYMMDDHHmmss").format(); - } else { - date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format(); - } - if (date == "Invalid date") { - date = moment(created).format(); - } - } else { - date = moment(created).format(); - } - } - asset.taken = asset.modified = date; - } - - let dst = picturesPath + path + "thumbs/" + file; - - return exists(dst).then(function(exist) { - if (exist) { - return; - } - - return image.resize(256, 256).withMetadata().toFile(dst).catch(function(error) { - setStatus("Error resizing image: " + dst + "\n" + error, "error"); - throw error; - }); - }).then(function() { - let dst = picturesPath + path + "thumbs/scaled/" + file; - return exists(dst).then(function(exist) { - if (exist) { - return; - } - - return image.resize(Math.min(1024, metadata.width)).withMetadata().toFile(dst).catch(function(error) { - setStatus("Error resizing image: " + dst + "\n" + error, "error"); - throw error; - }); - }); - }).then(function() { - return photoDB.sequelize.query("UPDATE photos SET " + - "added=:added,modified=:modified,taken=:taken,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " + - "WHERE id=:id", { - replacements: asset, - }); - }); - }).catch(function(error) { - setStatus("Error reading image " + src + ":\n" + error, "error"); - return moveCorrupt(path, file); - }); - }).then(function() { - toProcess--; - if (moment().add(-5, 'seconds') > lastMessage) { - setStatus("Items to be processed: " + toProcess); - lastMessage = moment(); - } - }); - }); - }).catch(function(error) { - setStatus("Error processing file. Continuing.", "error"); - throw error; - }).then(function() { - setStatus("Completed processing queue. Marking " + duplicates.length + " duplicates."); - return photoDB.sequelize.transaction(function(transaction) { - return Promise.mapSeries(duplicates, function(asset) { - return photoDB.sequelize.query("UPDATE photos " + - "SET duplicate=:duplicate,modified=CURRENT_TIMESTAMP,scanned=CURRENT_TIMESTAMP WHERE id=:id", { - replacements: asset, - transaction: transaction - }); - }); - }); - }).then(function() { - setStatus("Looking for removed assets"); - return photoDB.sequelize.query("SELECT photos.scanned,photos.id,photos.filename,albums.path FROM photos " + - "LEFT JOIN albums ON (albums.id=photos.albumId) " + - "WHERE photos.deleted=0 AND (DATETIME(photos.scanned) { - for (var i = 0; i < files.length; i++) { - /* If this file has an original NEF/ORF on the system, don't add the JPG to the DB */ - if (rawExtension.exec(files[i]) && file == files[i].replace(rawExtension, ".jpg")) { - return false; - } - - /* If there is a different CASE (eg. JPG vs jpg) don't add it, and remove the 'lower case' - * version from disk. */ - if (file != files[i] && file.toUpperCase() == files[i]) { - removeNewerFile(path, file, files[i]); - setStatus("Duplicate file in " + path + ": ", file, files[i]); - return false; - } - } - return metaDirectories.indexOf(file) == -1; - }); - - return resolve(files); - }); - }).then(function(files) { - return Promise.map(files, function(file) { - let filepath = path + file; - return stat(filepath).then(function(stats) { - if (stats.isDirectory()) { - filepath += "/"; - return scanDir(album, filepath).spread(function(_albums, _assets) { - album.allAssetCount += _assets.length; - album.allAlbumCount += _albums.length + 1; - albums = albums.concat(_albums); - assets = assets.concat(_assets); - }).catch(function(error) { - setStatus("Could not scanDir " + filepath + ": " + error, "error"); - }); - } - - /* Check file extensions */ - if (!re.exec(file)) { - return; - } - - album.hasAssets = true; - - const asset = { - filename: file.replace(rawExtension, ".jpg"), - name: file.replace(/.[^.]*$/, ""), - stats: { - mtime: stats.mtime, - ctime: stats.ctime - }, - size: stats.size, - album: album - } - if (file != asset.filename) { - asset.raw = file; - } - assets.push(asset); - }); - }); - }).then(function() { - return Promise.map(albums, function(album) { - if (album.hasAssets) { - return mkdir(album.path + "thumbs/scaled"); - } - }); - }).then(function() { - return [ albums, assets ]; - }); -} - -function findOrCreateDBAlbum(transaction, album) { - let query = "SELECT id FROM albums WHERE path=:path AND "; - if (!album.parent) { - query += "parentId IS NULL"; - album.parentId = null; - } else { - if (!album.parent.id) { - let error = "Albums in array in non ancestral order!"; - setStatus(error, "error"); - throw error; - } - album.parentId = album.parent.id; - query += "parentId=:parentId"; - } - - return photoDB.sequelize.query(query, { - replacements: album, - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(results) { - if (results.length == 0) { - if (!album.parent) { - setStatus("Creating top level album: " + picturesPath, "warn" ); - } - return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", { - replacements: album, - transaction: transaction - }).spread(function(results, metadata) { - return metadata.lastID; - }); - } else { - return results[0].id; - } - }).then(function(id) { - album.id = id; - return id; - }); -} - -function findOrUpdateDBAsset(transaction, asset) { - if (!asset.album || !asset.album.id) { - let error = "Asset being processed without an album"; - setStatus(error, "warn"); - throw error; - } - - asset.albumId = asset.album.id; - - return photoDB.sequelize.query( - "SELECT id,DATETIME(scanned) AS scanned,size,DATETIME(modified) AS modified " + - "FROM photos " + - "WHERE albumId=:albumId AND filename=:filename", { - replacements: asset, - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(results) { - if (results.length == 0) { - return photoDB.sequelize.query("INSERT INTO photos " + - "(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", { - replacements: asset, - transaction: transaction - }).spread(function(results, metadata) { - asset.id = metadata.lastID; - }); - } - - asset.id = results[0].id; - asset.scanned = new Date(results[0].scanned); - asset.modified = new Date(results[0].modified); - - /* If the size on disk changed, update the size entry in the DB. This shouldn't happen in - * production unless someone modifies the file, then re-stamps the modified time */ - if (asset.size != results[0].size) { - setStatus("File was modified with time-restamp (HASH regeneration will be queued): " + asset.filename); - delete asset.scanned; - delete asset.modified; - } - }).then(function() { - return asset; - }); -} - -function computeHash(filepath) { - return new Promise(function(resolve, reject) { - let input = fs.createReadStream(filepath), - hash = crypto.createHash("sha256"); - - if (!input) { - console.warn("Unable to open " + filepath); - return reject(); - } - - input.on("error", function(error) { - console.warn("Error reading " + filepath); - reject(error); - }); - - input.on("readable", function() { - const data = input.read(); - if (data) { - hash.update(data); - } else { - input.close(); - resolve(hash.digest("hex")); - hash = null; - input = null; - } - }); - }); -} - -let scanningStatus = []; - -function setStatus(status, level) { - if (status == "idle") { - scanningStatus = []; - return; - } - - level = level || "info"; - scanningStatus.push({ - level: level, - time: moment().format(), - log: status - }); - switch (level) { - case "error": - console.error(status); - break; - case "warn": - console.warn(status); - break; - default: - console.log(status); - } -} - -function doScan() { - /* 1. Scan for all assets which will be managed by the system. readdir - * 2. Check if entry in DB. Check mod-time in DB vs. stats from #1 - * - For albums - * - For assets - * 3. If not in DB, or mod-time changed, queue for HASH CHECK - * - * HASH CHECK - * 1. Compute HASH - * 2. Check for HASH in photohash -- skip? - * 3. Check for and create thumbs/FILE thumbs/scaled/FILE - * 4. If necessary, create JPG from RAW - * 5. Update last-scanned date in DB for entry - * 6. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were - * removed on disk)? Also purge from the HASH table. - */ - let initialized = Date.now(); - let now = Date.now(); - let needsProcessing = []; - - if (scanningStatus.length != 0) { - return Promise.resolve(scanningStatus); - } - - return scanDir(null, picturesPath).spread(function(albums, assets) { - setStatus("Found " + assets.length + " assets in " + albums.length + " albums after " + - ((Date.now() - now) / 1000) + "s"); - /* One at a time, in series, as the album[] array has parents first, then descendants. - * Operating in parallel could result in a child being searched for prior to the parent */ - now = Date.now(); - - let toProcess = albums.length, lastMessage = moment(); - return photoDB.sequelize.transaction(function(transaction) { - return Promise.mapSeries(albums, function(album) { - return findOrCreateDBAlbum(transaction, album).then(function() { - toProcess--; - if (moment().add(-5, 'seconds') > lastMessage) { - setStatus("Albums to be created in DB: " + toProcess); - lastMessage = moment(); - } - }); - }); - }).then(function() { - setStatus("Processed " + albums.length + " album DB entries in " + - ((Date.now() - now) / 1000) + "s"); - now = Date.now(); - - setStatus(assets.length + " assets remaining to be verified/updated. ETA N/A"); - - let processed = 0, start = Date.now(), last = 0, updateScanned = [], newEntries = 0; - return photoDB.sequelize.transaction(function(transaction) { - return Promise.map(assets, function(asset) { - return Promise.resolve(asset).then(function(asset) { - /* If both mtime and ctime of the asset are older than the lastScan, skip it - * - * Can only do this after a full scan has occurred */ - if (lastScan != null && asset.stats.mtime < lastScan && asset.stats.ctime < lastScan) { - return asset; - } - - return findOrUpdateDBAsset(transaction, asset).then(function(asset) { - if (!asset.scanned) { - newEntries++; - } - if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) { -// if (!asset.scanned) { console.log("no scan date on asset"); } -// if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); } -// if (!asset.modified) { console.log("no mtime."); } - needsProcessing.push(asset); - } else { - updateScanned.push(asset.id); - } - return asset; - }).then(function(asset) { - return asset; - }); - }).then(function(asset) { - processed++; - - let elapsed = Date.now() - start; - if (elapsed < 5000) { - return asset; - } - - let remaining = assets.length - processed, eta = Math.ceil((elapsed / 1000) * remaining / (processed - last)); - setStatus(remaining + " assets remaining be verified/updated " + - "(" + newEntries + " new entries, " + needsProcessing.length + " need processing," + (processed - newEntries) + " up-to-date so far). ETA " + eta + "s"); - last = processed; - start = Date.now(); - }); - }, { - concurrency: 10 - }); - }).then(function() { - if (updateScanned.length) { - return photoDB.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:ids)", { - replacements: { - ids: updateScanned - } - }).then(function() { - setStatus("Updated scan date of " + updateScanned.length + " assets"); - updateScanned = []; - }); - } - }).then(function() { - setStatus(newEntries + " assets are new. " + (needsProcessing.length - newEntries) + " assets have been modified.\n" + - needsProcessing.length + " assets need HASH computed. " + (assets.length - needsProcessing.length) + " need no update.");; - processBlock(needsProcessing); - needsProcessing = []; - }).then(function() { - setStatus("Scanned " + assets.length + " asset DB entries in " + - ((Date.now() - now) / 1000) + "s"); - assets = []; - }); - }); - }).then(function() { - setStatus("Total time to initialize DB and all scans: " + ((Date.now() - initialized) / 1000) + "s"); - return photoDB.sequelize.query("SELECT max(scanned) AS scanned FROM photos", { - type: photoDB.sequelize.QueryTypes.SELECT - }).then(function(results) { - if (results[0].scanned == null) { - lastScan = new Date("1800-01-01"); - } else { - lastScan = new Date(results[0].scanned); - } - setStatus("Updating any asset newer than " + moment(lastScan).format()); - }); - }).then(function() { - setStatus("idle"); - return "scan complete"; - }).catch(function(error) { - setStatus(error); - throw error; - }); -} - -module.exports = { - init: function(db) { - photoDB = db; - }, - scan: doScan -}; - diff --git a/package.json b/package.json index 3c6d8c3..019973f 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,12 @@ "description": "Settlers", "main": "index.html", "scripts": { + "start": "NODE_CONFIG_ENV='devel' node server/app.js", + "backend": "NODE_CONFIG_ENV='production' node server/app.js" "start": "webpack-dev-server --mode development --host 0.0.0.0 --config webpack.dev.js", "build": "webpack --config webpack.prod.js", "watch": "webpack --config webpack.prod.js --watch", - "backend": "NODE_CONFIG_ENV='production' node ketr.ketran/server/app.js" + "backend": "NODE_CONFIG_ENV='production' node server/app.js" }, "repository": "ssh://git@gitlab.ketrenos.com/jketreno/ketr.settlers.git", "author": "James Ketrenos ", @@ -33,6 +35,20 @@ "react-router-dom": "^5.0.1", "react-scroll": "^1.7.14", "react-syntax-highlighter": "^11.0.2" + "bluebird": "^3.5.5", + "config": "^3.1.0", + "connect-sqlite3": "^0.9.11", + "cookie-parser": "^1.4.4", + "express": "^4.17.1", + "express-session": "^1.17.1", + "handlebars": "^4.7.6", + "moment": "^2.24.0", + "morgan": "^1.9.1", + "node-fetch": "^2.6.0", + "node-gzip": "^1.1.2", + "nodemailer": "^6.3.0", + "sequelize": "^5.21.6", + "sqlite3": "^4.1.1" }, "devDependencies": { "@babel/cli": "^7.1.0", diff --git a/ketr.ketran/server/app.js b/server/app.js similarity index 97% rename from ketr.ketran/server/app.js rename to server/app.js index 65201a8..0bc2374 100755 --- a/ketr.ketran/server/app.js +++ b/server/app.js @@ -15,7 +15,8 @@ const express = require("express"), require("./console-line.js"); /* Monkey-patch console.log with line numbers */ -const serverConfig = config.get("server"); +const frontEndPath = config.get("frontendPath").replace(/\/$/, "") + "/", + serverConfig = config.get("server"); let basePath = config.get("basePath"); basePath = "/" + basePath.replace(/^\/+/, "").replace(/\/+$/, "") + "/"; @@ -255,7 +256,7 @@ app.use(basePath, function(req, res, next) { }); /* Everything below here requires a successful authentication */ -app.use(basePath, express.static(picturesPath, { index: false })); +app.use(basePath, express.static(frontendPath, { index: false })); app.use(basePath + "api/v1/games", require("./routes/games")); diff --git a/ketr.ketran/server/console-line.js b/server/console-line.js similarity index 100% rename from ketr.ketran/server/console-line.js rename to server/console-line.js diff --git a/ketr.ketran/server/db/games.db b/server/db/games.db similarity index 100% rename from ketr.ketran/server/db/games.db rename to server/db/games.db diff --git a/ketr.ketran/server/db/users.db b/server/db/users.db similarity index 100% rename from ketr.ketran/server/db/users.db rename to server/db/users.db diff --git a/ketr.ketran/server/http-server.js b/server/http-server.js similarity index 100% rename from ketr.ketran/server/http-server.js rename to server/http-server.js diff --git a/ketr.ketran/server/lib/mail.js b/server/lib/mail.js similarity index 100% rename from ketr.ketran/server/lib/mail.js rename to server/lib/mail.js diff --git a/ketr.ketran/server/lib/pascha-dates.js b/server/lib/pascha-dates.js similarity index 100% rename from ketr.ketran/server/lib/pascha-dates.js rename to server/lib/pascha-dates.js diff --git a/ketr.ketran/server/lib/pascha.js b/server/lib/pascha.js similarity index 100% rename from ketr.ketran/server/lib/pascha.js rename to server/lib/pascha.js diff --git a/ketr.ketran/server/mail.js b/server/mail.js similarity index 100% rename from ketr.ketran/server/mail.js rename to server/mail.js diff --git a/ketr.ketran/server/routes/.identities.js.swp b/server/routes/.identities.js.swp similarity index 100% rename from ketr.ketran/server/routes/.identities.js.swp rename to server/routes/.identities.js.swp diff --git a/ketr.ketran/server/routes/basepath.js b/server/routes/basepath.js similarity index 100% rename from ketr.ketran/server/routes/basepath.js rename to server/routes/basepath.js diff --git a/ketr.ketran/server/routes/games.js b/server/routes/games.js similarity index 100% rename from ketr.ketran/server/routes/games.js rename to server/routes/games.js diff --git a/ketr.ketran/server/routes/index.js b/server/routes/index.js similarity index 100% rename from ketr.ketran/server/routes/index.js rename to server/routes/index.js diff --git a/ketr.ketran/server/routes/users.js b/server/routes/users.js similarity index 100% rename from ketr.ketran/server/routes/users.js rename to server/routes/users.js diff --git a/ketr.ketran/server/timestamp.js b/server/timestamp.js similarity index 100% rename from ketr.ketran/server/timestamp.js rename to server/timestamp.js diff --git a/webpack.dev.js b/webpack.dev.js index d55dfa0..3cd9a92 100644 --- a/webpack.dev.js +++ b/webpack.dev.js @@ -7,8 +7,8 @@ module.exports = merge(common, { mode: "development", devServer: { contentBase: path.join(__dirname, "/"), - port: 8008, - publicPath: "http://localhost:8008/dist/", + port: 8930, + publicPath: "http://localhost:8930/dist/", hotOnly: true, disableHostCheck: true, historyApiFallback: true