Restructured directory arrangement
Signed-off-by: James Ketrenos <james.p.ketrenos@intel.com>
This commit is contained in:
James Ketrenos
parent
dca37b1723
commit
ddc9eccfc7
75
README.md
75
README.md
@ -1,8 +1,81 @@
|
|||||||
# Ketr Settlers
|
# Ketr Settlers
|
||||||
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo apt-get install nodejs npm
|
sudo apt-get install nodejs npm
|
||||||
sudo -E npm install --global npm@latest
|
sudo -E npm install --global npm@latest
|
||||||
```
|
```
|
||||||
|
|
||||||
|
# Ketr.Ketran REST API
|
||||||
|
|
||||||
|
## POST /api/v1/game
|
||||||
|
|
||||||
|
### Request
|
||||||
|
|
||||||
|
```json
|
||||||
|
{}
|
||||||
|
````
|
||||||
|
|
||||||
|
### Response
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
gameId: id
|
||||||
|
gameState: {
|
||||||
|
tiles: []
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
# Configuring / installing
|
||||||
|
|
||||||
|
|
||||||
|
## Build
|
||||||
|
```bash
|
||||||
|
git clone ...
|
||||||
|
cd server
|
||||||
|
npm install
|
||||||
|
```
|
||||||
|
|
||||||
|
## Install
|
||||||
|
```bash
|
||||||
|
sudo cp ketr.ketran /etc/logrotate.d/
|
||||||
|
sudo cp ketr.ketran.service /etc/systemd/system/
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
```
|
||||||
|
|
||||||
|
Install the following into your nginx server configuration:
|
||||||
|
|
||||||
|
```nginx
|
||||||
|
location ~ /ketr.ketran/api/.* {
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-NginX-Proxy true;
|
||||||
|
proxy_pass_header Set-Cookie;
|
||||||
|
proxy_pass_header P3P;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
proxy_pass http://localhost:8930;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Add security tokens in ketr.ketran/config/local.json:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cat << EOF > config/local.json
|
||||||
|
{
|
||||||
|
"tokens": [ {
|
||||||
|
"$(whoami)": "$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo;)"
|
||||||
|
} ]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
## Launch
|
||||||
|
```bash
|
||||||
|
sudo systemctl start ketr.ketran
|
||||||
|
```
|
||||||
|
|
||||||
|
|||||||
5
config/local.json
Normal file
5
config/local.json
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"tokens": [ {
|
||||||
|
"jketreno": "1MhGsldnwNkH9d2s-yu8fxZ0JcHCpClY"
|
||||||
|
} ]
|
||||||
|
}
|
||||||
0
ketr.ketran/.gitignore → f/.gitignore
vendored
0
ketr.ketran/.gitignore → f/.gitignore
vendored
@ -1,74 +0,0 @@
|
|||||||
# Ketr.Ketran REST API
|
|
||||||
|
|
||||||
## POST /api/v1/game
|
|
||||||
|
|
||||||
### Request
|
|
||||||
|
|
||||||
```json
|
|
||||||
{}
|
|
||||||
````
|
|
||||||
|
|
||||||
### Response
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
gameId: id
|
|
||||||
gameState: {
|
|
||||||
tiles: []
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
# Configuring / installing
|
|
||||||
|
|
||||||
|
|
||||||
## Build
|
|
||||||
```bash
|
|
||||||
git clone ...
|
|
||||||
cd server
|
|
||||||
npm install
|
|
||||||
```
|
|
||||||
|
|
||||||
## Install
|
|
||||||
```bash
|
|
||||||
sudo cp ketr.ketran /etc/logrotate.d/
|
|
||||||
sudo cp ketr.ketran.service /etc/systemd/system/
|
|
||||||
sudo systemctl daemon-reload
|
|
||||||
```
|
|
||||||
|
|
||||||
Install the following into your nginx server configuration:
|
|
||||||
|
|
||||||
```nginx
|
|
||||||
location ~ /ketr.ketran/api/.* {
|
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
|
||||||
proxy_set_header Host $http_host;
|
|
||||||
proxy_set_header X-NginX-Proxy true;
|
|
||||||
proxy_pass_header Set-Cookie;
|
|
||||||
proxy_pass_header P3P;
|
|
||||||
proxy_http_version 1.1;
|
|
||||||
proxy_set_header Upgrade $http_upgrade;
|
|
||||||
proxy_set_header Connection "upgrade";
|
|
||||||
proxy_pass http://localhost:8930;
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Add security tokens in ketr.ketran/config/local.json:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cat << EOF > config/local.json
|
|
||||||
{
|
|
||||||
"tokens": [ {
|
|
||||||
"$(whoami)": "$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo;)"
|
|
||||||
} ]
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
## Launch
|
|
||||||
```bash
|
|
||||||
sudo systemctl start ketr.ketran
|
|
||||||
```
|
|
||||||
|
|
||||||
@ -1,569 +0,0 @@
|
|||||||
[
|
|
||||||
{
|
|
||||||
"directory": true,
|
|
||||||
"date_modified": "2020-03-30T18:29:16+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux",
|
|
||||||
"size": 4096,
|
|
||||||
"path": "linux"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": true,
|
|
||||||
"date_modified": "2020-03-30T18:29:16+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu",
|
|
||||||
"size": 4096,
|
|
||||||
"path": "linux/ubuntu"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": true,
|
|
||||||
"date_modified": "2020-03-30T18:29:23+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10",
|
|
||||||
"size": 8192,
|
|
||||||
"path": "linux/ubuntu/19.10"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:15:08+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-cmc-2.0+embargo-169.u19.10-release.x86_64.deb",
|
|
||||||
"size": 31039028,
|
|
||||||
"path": "linux/ubuntu/19.10/intel-cmc-2.0+embargo-169.u19.10-release.x86_64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:22:21+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-gpu-tools-dbg_1.24+embargo169_amd64.deb",
|
|
||||||
"size": 6465620,
|
|
||||||
"path": "linux/ubuntu/19.10/intel-gpu-tools-dbg_1.24+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:22:21+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-gpu-tools_1.24+embargo169_amd64.deb",
|
|
||||||
"size": 1955740,
|
|
||||||
"path": "linux/ubuntu/19.10/intel-gpu-tools_1.24+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:13:12+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-level-zero-gpu_0.8.016262+embargo169_amd64.deb",
|
|
||||||
"size": 1107520,
|
|
||||||
"path": "linux/ubuntu/19.10/intel-level-zero-gpu_0.8.016262+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:35:29+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-media-va-driver-non-free_20.1~pre+embargo169_amd64.deb",
|
|
||||||
"size": 6244784,
|
|
||||||
"path": "linux/ubuntu/19.10/intel-media-va-driver-non-free_20.1~pre+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:18:25+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-metrics-discovery_1.5.114+embargo169_amd64.deb",
|
|
||||||
"size": 715152,
|
|
||||||
"path": "linux/ubuntu/19.10/intel-metrics-discovery_1.5.114+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:19:21+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-metrics-library_1.0.1+embargo169_amd64.deb",
|
|
||||||
"size": 157448,
|
|
||||||
"path": "linux/ubuntu/19.10/intel-metrics-library_1.0.1+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:57:35+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/intel-opencl-icd_016320+embargo169_amd64.deb",
|
|
||||||
"size": 1223400,
|
|
||||||
"path": "linux/ubuntu/19.10/intel-opencl-icd_016320+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:57:52+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/level-zero-dev_0.91.7+embargo169_amd64.deb",
|
|
||||||
"size": 90364,
|
|
||||||
"path": "linux/ubuntu/19.10/level-zero-dev_0.91.7+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:57:52+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/level-zero_0.91.7+embargo169_amd64.deb",
|
|
||||||
"size": 49700,
|
|
||||||
"path": "linux/ubuntu/19.10/level-zero_0.91.7+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:36+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-amdgpu1_2.4.100+embargo169_amd64.deb",
|
|
||||||
"size": 28272,
|
|
||||||
"path": "linux/ubuntu/19.10/libdrm-amdgpu1_2.4.100+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:36+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-common_2.4.100+embargo169_all.deb",
|
|
||||||
"size": 14208,
|
|
||||||
"path": "linux/ubuntu/19.10/libdrm-common_2.4.100+embargo169_all.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:36+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-dev_2.4.100+embargo169_amd64.deb",
|
|
||||||
"size": 120324,
|
|
||||||
"path": "linux/ubuntu/19.10/libdrm-dev_2.4.100+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:36+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-intel1_2.4.100+embargo169_amd64.deb",
|
|
||||||
"size": 70724,
|
|
||||||
"path": "linux/ubuntu/19.10/libdrm-intel1_2.4.100+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:36+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-nouveau2_2.4.100+embargo169_amd64.deb",
|
|
||||||
"size": 26316,
|
|
||||||
"path": "linux/ubuntu/19.10/libdrm-nouveau2_2.4.100+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:36+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm-radeon1_2.4.100+embargo169_amd64.deb",
|
|
||||||
"size": 29580,
|
|
||||||
"path": "linux/ubuntu/19.10/libdrm-radeon1_2.4.100+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:36+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libdrm2_2.4.100+embargo169_amd64.deb",
|
|
||||||
"size": 41172,
|
|
||||||
"path": "linux/ubuntu/19.10/libdrm2_2.4.100+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libegl-mesa0_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 139724,
|
|
||||||
"path": "linux/ubuntu/19.10/libegl-mesa0_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libegl1-mesa-dev_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 51120,
|
|
||||||
"path": "linux/ubuntu/19.10/libegl1-mesa-dev_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libegl1-mesa_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 49392,
|
|
||||||
"path": "linux/ubuntu/19.10/libegl1-mesa_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgbm-dev_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 52940,
|
|
||||||
"path": "linux/ubuntu/19.10/libgbm-dev_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgbm1_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 71004,
|
|
||||||
"path": "linux/ubuntu/19.10/libgbm1_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgl1-mesa-dev_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 49408,
|
|
||||||
"path": "linux/ubuntu/19.10/libgl1-mesa-dev_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgl1-mesa-dri_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 9929788,
|
|
||||||
"path": "linux/ubuntu/19.10/libgl1-mesa-dri_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgl1-mesa-glx_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 49412,
|
|
||||||
"path": "linux/ubuntu/19.10/libgl1-mesa-glx_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libglapi-mesa_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 69888,
|
|
||||||
"path": "linux/ubuntu/19.10/libglapi-mesa_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgles2-mesa-dev_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 49416,
|
|
||||||
"path": "linux/ubuntu/19.10/libgles2-mesa-dev_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libgles2-mesa_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 49400,
|
|
||||||
"path": "linux/ubuntu/19.10/libgles2-mesa_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libglx-mesa0_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 182580,
|
|
||||||
"path": "linux/ubuntu/19.10/libglx-mesa0_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:48:07+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigc-dev_8020561+embargo169_amd64.deb",
|
|
||||||
"size": 2376,
|
|
||||||
"path": "linux/ubuntu/19.10/libigc-dev_8020561+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:48:07+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigc-tools_8020561+embargo169_amd64.deb",
|
|
||||||
"size": 1830108,
|
|
||||||
"path": "linux/ubuntu/19.10/libigc-tools_8020561+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:48:07+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigc1_8020561+embargo169_amd64.deb",
|
|
||||||
"size": 11786756,
|
|
||||||
"path": "linux/ubuntu/19.10/libigc1_8020561+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:48:07+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigdfcl-dev_8020561+embargo169_amd64.deb",
|
|
||||||
"size": 125484,
|
|
||||||
"path": "linux/ubuntu/19.10/libigdfcl-dev_8020561+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:48:07+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigdfcl1_8020561+embargo169_amd64.deb",
|
|
||||||
"size": 18982892,
|
|
||||||
"path": "linux/ubuntu/19.10/libigdfcl1_8020561+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:45+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigdgmm-dev_8020561+embargo169_amd64.deb",
|
|
||||||
"size": 1126848,
|
|
||||||
"path": "linux/ubuntu/19.10/libigdgmm-dev_8020561+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:45+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigdgmm6_8020561+embargo169_amd64.deb",
|
|
||||||
"size": 127364,
|
|
||||||
"path": "linux/ubuntu/19.10/libigdgmm6_8020561+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:35:29+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigfxcmrt-dev_20.1~pre+embargo169_amd64.deb",
|
|
||||||
"size": 69096,
|
|
||||||
"path": "linux/ubuntu/19.10/libigfxcmrt-dev_20.1~pre+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:35:29+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libigfxcmrt7_20.1~pre+embargo169_amd64.deb",
|
|
||||||
"size": 30668,
|
|
||||||
"path": "linux/ubuntu/19.10/libigfxcmrt7_20.1~pre+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:17:58+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libmfx-dev_20.1~pre+embargo169_amd64.deb",
|
|
||||||
"size": 39952,
|
|
||||||
"path": "linux/ubuntu/19.10/libmfx-dev_20.1~pre+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:17:58+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libmfx-tools_20.1~pre+embargo169_amd64.deb",
|
|
||||||
"size": 1473148,
|
|
||||||
"path": "linux/ubuntu/19.10/libmfx-tools_20.1~pre+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:17:58+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libmfx1_20.1~pre+embargo169_amd64.deb",
|
|
||||||
"size": 2833028,
|
|
||||||
"path": "linux/ubuntu/19.10/libmfx1_20.1~pre+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libosmesa6-dev_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 52776,
|
|
||||||
"path": "linux/ubuntu/19.10/libosmesa6-dev_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libosmesa6_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 2741820,
|
|
||||||
"path": "linux/ubuntu/19.10/libosmesa6_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:23:06+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva-dev_2.7.0.1+embargo169_amd64.deb",
|
|
||||||
"size": 104500,
|
|
||||||
"path": "linux/ubuntu/19.10/libva-dev_2.7.0.1+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:23:06+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva-drm2_2.7.0.1+embargo169_amd64.deb",
|
|
||||||
"size": 17760,
|
|
||||||
"path": "linux/ubuntu/19.10/libva-drm2_2.7.0.1+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:23:06+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva-glx2_2.7.0.1+embargo169_amd64.deb",
|
|
||||||
"size": 21344,
|
|
||||||
"path": "linux/ubuntu/19.10/libva-glx2_2.7.0.1+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:23:06+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva-wayland2_2.7.0.1+embargo169_amd64.deb",
|
|
||||||
"size": 19664,
|
|
||||||
"path": "linux/ubuntu/19.10/libva-wayland2_2.7.0.1+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:23:06+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva-x11-2_2.7.0.1+embargo169_amd64.deb",
|
|
||||||
"size": 22392,
|
|
||||||
"path": "linux/ubuntu/19.10/libva-x11-2_2.7.0.1+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:23:06+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libva2_2.7.0.1+embargo169_amd64.deb",
|
|
||||||
"size": 60736,
|
|
||||||
"path": "linux/ubuntu/19.10/libva2_2.7.0.1+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libwayland-egl1-mesa_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 49416,
|
|
||||||
"path": "linux/ubuntu/19.10/libwayland-egl1-mesa_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libxatracker-dev_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 52896,
|
|
||||||
"path": "linux/ubuntu/19.10/libxatracker-dev_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/libxatracker2_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 1644200,
|
|
||||||
"path": "linux/ubuntu/19.10/libxatracker2_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:20+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/linux-headers-5.4.17-169+_5.4.17-169+-1_amd64.deb",
|
|
||||||
"size": 11420780,
|
|
||||||
"path": "linux/ubuntu/19.10/linux-headers-5.4.17-169+_5.4.17-169+-1_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T16:55:17+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/linux-i915-firmware_2020.12+embargo169.deb",
|
|
||||||
"size": 4758832,
|
|
||||||
"path": "linux/ubuntu/19.10/linux-i915-firmware_2020.12+embargo169.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:21+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/linux-image-5.4.17-169+-dbg_5.4.17-169+-1_amd64.deb",
|
|
||||||
"size": 903956216,
|
|
||||||
"path": "linux/ubuntu/19.10/linux-image-5.4.17-169+-dbg_5.4.17-169+-1_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:21+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/linux-image-5.4.17-169+_5.4.17-169+-1_amd64.deb",
|
|
||||||
"size": 60596400,
|
|
||||||
"path": "linux/ubuntu/19.10/linux-image-5.4.17-169+_5.4.17-169+-1_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:20:21+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/linux-libc-dev_5.4.17-169+-1_amd64.deb",
|
|
||||||
"size": 1069796,
|
|
||||||
"path": "linux/ubuntu/19.10/linux-libc-dev_5.4.17-169+-1_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T16:55:07+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/manifest.yml",
|
|
||||||
"size": 5595,
|
|
||||||
"path": "linux/ubuntu/19.10/manifest.yml"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/mesa-common-dev_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 677236,
|
|
||||||
"path": "linux/ubuntu/19.10/mesa-common-dev_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/mesa-opencl-icd_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 10299076,
|
|
||||||
"path": "linux/ubuntu/19.10/mesa-opencl-icd_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/mesa-va-drivers_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 2684040,
|
|
||||||
"path": "linux/ubuntu/19.10/mesa-va-drivers_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/mesa-vdpau-drivers_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 2810196,
|
|
||||||
"path": "linux/ubuntu/19.10/mesa-vdpau-drivers_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:27:11+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/mesa-vulkan-drivers_20.1.0-devel+embargo169_amd64.deb",
|
|
||||||
"size": 3587736,
|
|
||||||
"path": "linux/ubuntu/19.10/mesa-vulkan-drivers_20.1.0-devel+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:23:06+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/va-driver-all_2.7.0.1+embargo169_amd64.deb",
|
|
||||||
"size": 13948,
|
|
||||||
"path": "linux/ubuntu/19.10/va-driver-all_2.7.0.1+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T17:23:06+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/linux/ubuntu/19.10/vainfo_2.6.0.0+embargo169_amd64.deb",
|
|
||||||
"size": 18908,
|
|
||||||
"path": "linux/ubuntu/19.10/vainfo_2.6.0.0+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": true,
|
|
||||||
"date_modified": "2020-03-30T18:29:26+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils",
|
|
||||||
"size": 4096,
|
|
||||||
"path": "utils"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": true,
|
|
||||||
"date_modified": "2020-03-30T18:29:26+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu",
|
|
||||||
"size": 4096,
|
|
||||||
"path": "utils/ubuntu"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": true,
|
|
||||||
"date_modified": "2020-03-30T18:29:27+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10",
|
|
||||||
"size": 4096,
|
|
||||||
"path": "utils/ubuntu/19.10"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:29:15+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/crucible_1.0.20200316+i169_amd64.deb",
|
|
||||||
"size": 14641300,
|
|
||||||
"path": "utils/ubuntu/19.10/crucible_1.0.20200316+i169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:20:14+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/libwaffle-1-0_1.6.90+embargo169_amd64.deb",
|
|
||||||
"size": 28100,
|
|
||||||
"path": "utils/ubuntu/19.10/libwaffle-1-0_1.6.90+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:20:14+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/libwaffle-dev_1.6.90+embargo169_amd64.deb",
|
|
||||||
"size": 7444,
|
|
||||||
"path": "utils/ubuntu/19.10/libwaffle-dev_1.6.90+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:20:14+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/libwaffle-doc_1.6.90+embargo169_all.deb",
|
|
||||||
"size": 4504,
|
|
||||||
"path": "utils/ubuntu/19.10/libwaffle-doc_1.6.90+embargo169_all.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:29:15+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/piglit-shards_20200330+embargo169.deb",
|
|
||||||
"size": 15768,
|
|
||||||
"path": "utils/ubuntu/19.10/piglit-shards_20200330+embargo169.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:28:06+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/piglit_1.0.20200212+embargo169_amd64.deb",
|
|
||||||
"size": 33672144,
|
|
||||||
"path": "utils/ubuntu/19.10/piglit_1.0.20200212+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:20:14+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/utils/ubuntu/19.10/waffle-utils_1.6.90+embargo169_amd64.deb",
|
|
||||||
"size": 11236,
|
|
||||||
"path": "utils/ubuntu/19.10/waffle-utils_1.6.90+embargo169_amd64.deb"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": true,
|
|
||||||
"date_modified": "2020-03-30T18:29:27+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/TestTools",
|
|
||||||
"size": 4096,
|
|
||||||
"path": "TestTools"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": true,
|
|
||||||
"date_modified": "2020-03-30T18:29:27+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/TestTools/Linux",
|
|
||||||
"size": 4096,
|
|
||||||
"path": "TestTools/Linux"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"directory": false,
|
|
||||||
"date_modified": "2020-03-30T18:29:16+0000",
|
|
||||||
"url": "https://ubitstore.intel.com/webstores/fm/sfa/Artifacts/Graphics/Builds/cogd/dynamic/run/builds/b561/8020561/artifacts/TestTools/Linux/TestTools-Release-Internal-x64.tar.xz",
|
|
||||||
"size": 368,
|
|
||||||
"path": "TestTools/Linux/TestTools-Release-Internal-x64.tar.xz"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
@ -1,404 +0,0 @@
|
|||||||
/*
|
|
||||||
* Face recognition:
|
|
||||||
* 1. For each photo, extract all faces. Store face rectangles.
|
|
||||||
* face_id unique
|
|
||||||
* photo_id foreign key
|
|
||||||
* top left bottom right
|
|
||||||
* identity_id
|
|
||||||
* distance (0 == truth; manually assigned identity)
|
|
||||||
* 2. For each face_id, create:
|
|
||||||
* /${picturesPath}face-data/${face_id % 100}/
|
|
||||||
* ${face_id}-normalized
|
|
||||||
* ${face_id}-original
|
|
||||||
* ${face_id}-data
|
|
||||||
*/
|
|
||||||
|
|
||||||
"use strict";
|
|
||||||
|
|
||||||
process.env.TZ = "Etc/GMT";
|
|
||||||
|
|
||||||
console.log("Loading face-recognizer");
|
|
||||||
|
|
||||||
require('@tensorflow/tfjs-node');
|
|
||||||
|
|
||||||
const config = require("config"),
|
|
||||||
Promise = require("bluebird"),
|
|
||||||
{ exists, mkdir, unlink } = require("./lib/util"),
|
|
||||||
faceapi = require("face-api.js"),
|
|
||||||
fs = require("fs"),
|
|
||||||
canvas = require("canvas");
|
|
||||||
|
|
||||||
const { createCanvas, Canvas, Image, ImageData } = canvas;
|
|
||||||
|
|
||||||
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
|
|
||||||
|
|
||||||
const maxConcurrency = require("os").cpus().length;
|
|
||||||
|
|
||||||
require("./console-line.js"); /* Monkey-patch console.log with line numbers */
|
|
||||||
|
|
||||||
const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
|
|
||||||
faceData = picturesPath + "face-data/";
|
|
||||||
|
|
||||||
let photoDB = null;
|
|
||||||
|
|
||||||
console.log("Loading pictures out of: " + picturesPath);
|
|
||||||
|
|
||||||
function alignFromLandmarks(image, landmarks) {
|
|
||||||
const faceMargin = 0.3,
|
|
||||||
width = 256, height = 256,
|
|
||||||
dY = landmarks._positions[45]._y - landmarks._positions[36]._y,
|
|
||||||
dX = landmarks._positions[45]._x - landmarks._positions[36]._x,
|
|
||||||
mid = {
|
|
||||||
x: landmarks._positions[36]._x + 0.5 * dX,
|
|
||||||
y: landmarks._positions[36]._y + 0.5 * dY
|
|
||||||
},
|
|
||||||
rotation = -Math.atan2(dY, dX),
|
|
||||||
cosRotation = Math.cos(rotation),
|
|
||||||
sinRotation = Math.sin(rotation),
|
|
||||||
eyeDistance = Math.sqrt(dY * dY + dX * dX),
|
|
||||||
scale = width * (1.0 - 2. * faceMargin) / eyeDistance,
|
|
||||||
canvas = createCanvas(width, height),
|
|
||||||
ctx = canvas.getContext("2d");
|
|
||||||
|
|
||||||
const prime = {
|
|
||||||
x: mid.x * cosRotation - mid.y * sinRotation,
|
|
||||||
y: mid.y * cosRotation + mid.x * sinRotation
|
|
||||||
};
|
|
||||||
|
|
||||||
mid.x = prime.x;
|
|
||||||
mid.y = prime.y;
|
|
||||||
|
|
||||||
ctx.translate(
|
|
||||||
0.5 * width - mid.x * scale,
|
|
||||||
0.5 * height - (height * (0.5 - faceMargin)) - mid.y * scale);
|
|
||||||
ctx.rotate(rotation);
|
|
||||||
ctx.scale(scale, scale);
|
|
||||||
ctx.drawImage(image, 0, 0);
|
|
||||||
/*
|
|
||||||
ctx.strokeStyle = "red";
|
|
||||||
ctx.strokeWidth = "1";
|
|
||||||
ctx.beginPath();
|
|
||||||
landmarks._positions.forEach((point, index) => {
|
|
||||||
if (index == 0) {
|
|
||||||
ctx.moveTo(point._x, point._y);
|
|
||||||
} else {
|
|
||||||
ctx.lineTo(point._x, point._y);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
ctx.stroke();
|
|
||||||
*/
|
|
||||||
return canvas;
|
|
||||||
}
|
|
||||||
|
|
||||||
process.stdout.write("Loading DB.");
|
|
||||||
require("./db/photos").then(function(db) {
|
|
||||||
process.stdout.write("done\n");
|
|
||||||
photoDB = db;
|
|
||||||
}).then(() => {
|
|
||||||
console.log("DB connected.");
|
|
||||||
process.stdout.write("Loading models.");
|
|
||||||
return faceapi.nets.ssdMobilenetv1.loadFromDisk('./models');
|
|
||||||
}).then(() => {
|
|
||||||
process.stdout.write(".");
|
|
||||||
return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
|
|
||||||
}).then(() => {
|
|
||||||
process.stdout.write(".");
|
|
||||||
return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
|
|
||||||
}).then(() => {
|
|
||||||
console.log("Beginning face detection scanning.");
|
|
||||||
return photoDB.sequelize.query("SELECT photos.id,photos.filename,photos.width,photos.height,albums.path " +
|
|
||||||
"FROM photos " +
|
|
||||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
|
||||||
"WHERE faces=-1 AND photos.duplicate=0 AND photos.deleted=0 ORDER BY albums.path,photos.filename", {
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
||||||
raw: true
|
|
||||||
});
|
|
||||||
}).then((needToScan) => {
|
|
||||||
const total = needToScan.length;
|
|
||||||
let remaining = total,
|
|
||||||
processed = 0,
|
|
||||||
lastStatus = Date.now();
|
|
||||||
|
|
||||||
console.log(`${needToScan.length} photos have not had faces scanned.`);
|
|
||||||
|
|
||||||
return Promise.map(needToScan, (photo) => {
|
|
||||||
const photoPath = photo.path + photo.filename;
|
|
||||||
|
|
||||||
console.log(`Processing ${photoPath}...`);
|
|
||||||
|
|
||||||
/* Remove any existing face data for this photo */
|
|
||||||
return photoDB.sequelize.query("SELECT id FROM faces WHERE photoId=:id", {
|
|
||||||
replacements: photo,
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
||||||
raw: true
|
|
||||||
}).then((faces) => {
|
|
||||||
/* For each face-id, remove any face-data files, and then remove all the entries
|
|
||||||
* from the DB */
|
|
||||||
return Promise.map(faces, (face) => {
|
|
||||||
return Promise.map([ "-data.json", "-original.png" ], (suffix) => {
|
|
||||||
const id = face.id,
|
|
||||||
dataPath = faceData + (id % 100) + "/" + id + suffix;
|
|
||||||
return exists(dataPath).then((result) => {
|
|
||||||
if (result) {
|
|
||||||
console.log(`...removing ${dataPath}`);
|
|
||||||
return unlink(dataPath);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
return photoDB.sequelize.query("DELETE FROM faces WHERE photoId=:id", {
|
|
||||||
replacements: photo,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(async () => {
|
|
||||||
/* Process image for faces data */
|
|
||||||
const image = await canvas.loadImage(picturesPath + photoPath);
|
|
||||||
const detections = await faceapi.detectAllFaces(image,
|
|
||||||
new faceapi.SsdMobilenetv1Options({
|
|
||||||
minConfidence: 0.9
|
|
||||||
})
|
|
||||||
).withFaceLandmarks();
|
|
||||||
|
|
||||||
if (detections.length > 0) {
|
|
||||||
console.log(`...${detections.length} faces identified in ${photoPath}.`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return Promise.map(detections, async (face) => {
|
|
||||||
const detection = face.detection,
|
|
||||||
canvas = alignFromLandmarks(image, face.landmarks);
|
|
||||||
face.descriptor = await faceapi.computeFaceDescriptor(canvas);
|
|
||||||
|
|
||||||
const width = detection._box._width,
|
|
||||||
height = detection._box._height,
|
|
||||||
replacements = {
|
|
||||||
id: photo.id,
|
|
||||||
top: detection._box._y / detection._imageDims.height,
|
|
||||||
left: detection._box._x / detection._imageDims.width,
|
|
||||||
bottom: (detection._box._y + height) / detection._imageDims.height,
|
|
||||||
right: (detection._box._x + width) / detection._imageDims.width,
|
|
||||||
faceConfidence: detection._score
|
|
||||||
};
|
|
||||||
|
|
||||||
return photoDB.sequelize.query("INSERT INTO faces (photoId,top,left,bottom,right,faceConfidence) " +
|
|
||||||
"VALUES (:id,:top,:left,:bottom,:right,:faceConfidence)", {
|
|
||||||
replacements: replacements
|
|
||||||
}).spread((results, metadata) => {
|
|
||||||
return metadata.lastID;
|
|
||||||
}).then((id) => {
|
|
||||||
const path = faceData + (id % 100);
|
|
||||||
return mkdir(path).then(() => {
|
|
||||||
const dataPath = `${path}/${id}-data.json`, data = [];
|
|
||||||
console.log(`...writing descriptor data to ${dataPath}...`);
|
|
||||||
/* Confert from sparse object to dense array */
|
|
||||||
for (let i = 0; i < 128; i++) {
|
|
||||||
data.push(face.descriptor[i]);
|
|
||||||
}
|
|
||||||
fs.writeFileSync(dataPath, JSON.stringify(data));
|
|
||||||
}).then(() => {
|
|
||||||
const target = `${path}/${id}-original.png`;
|
|
||||||
console.log(`...writing aligned face crop to ${target}.`);
|
|
||||||
fs.writeFileSync(target, canvas.toBuffer("image/png", {
|
|
||||||
quality: 0.95,
|
|
||||||
chromaSubsampling: false
|
|
||||||
}));
|
|
||||||
}).catch((error) => {
|
|
||||||
console.error(error);
|
|
||||||
process.exit(-1);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
|
|
||||||
replacements: {
|
|
||||||
id: photo.id,
|
|
||||||
faces: detections.length
|
|
||||||
},
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).catch((error) => {
|
|
||||||
console.log(error);
|
|
||||||
console.warn("Skipping out on image " + photoPath + " and marking to 0 faces to prevent future scanning.");
|
|
||||||
return photoDB.sequelize.query("UPDATE photos SET faces=:faces WHERE id=:id", {
|
|
||||||
replacements: {
|
|
||||||
id: photo.id,
|
|
||||||
faces: 0
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
processed++;
|
|
||||||
const now = Date.now();
|
|
||||||
if (now - lastStatus > 5000) {
|
|
||||||
const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10,
|
|
||||||
eta = Math.round((total - processed) / rate);
|
|
||||||
lastStatus = now;
|
|
||||||
remaining = total - processed;
|
|
||||||
console.log(`Processing ${rate} images per second. ${remaining} images to be processed. ETA: ${eta}s`);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}, {
|
|
||||||
concurrency: maxConcurrency
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
console.log("Looking for face distances that need to be updated...");
|
|
||||||
let maxId;
|
|
||||||
|
|
||||||
return photoDB.sequelize.query("SELECT faces.id FROM faces ORDER BY faces.id DESC LIMIT 1", {
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
||||||
raw: true
|
|
||||||
}).then((results) => {
|
|
||||||
if (!results.length) {
|
|
||||||
console.log("...no faces exist yet to generate distances.");
|
|
||||||
maxId = 0;
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
maxId = results[0].id;
|
|
||||||
return photoDB.sequelize.query(
|
|
||||||
"SELECT faces.id,faces.lastComparedId " +
|
|
||||||
"FROM faces INNER JOIN photos ON photos.duplicate=0 AND photos.deleted=0 AND photos.id=faces.photoId " +
|
|
||||||
"WHERE faces.lastComparedId<:maxId OR faces.lastComparedId IS NULL " +
|
|
||||||
"ORDER BY faces.id ASC", {
|
|
||||||
replacements: {
|
|
||||||
maxId: maxId
|
|
||||||
},
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
||||||
raw: true
|
|
||||||
});
|
|
||||||
}).then((facesToUpdate) => {
|
|
||||||
console.log(`...${facesToUpdate.length} faces need distances updated.`);
|
|
||||||
console.log("---- run scanner/scanner !! ---");
|
|
||||||
return [];
|
|
||||||
if (facesToUpdate.length == 0) {
|
|
||||||
return facesToUpdate;
|
|
||||||
}
|
|
||||||
|
|
||||||
const descriptors = {};
|
|
||||||
|
|
||||||
return photoDB.sequelize.query(
|
|
||||||
"SELECT id FROM faces ORDER BY id ASC", {
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
||||||
raw: true
|
|
||||||
}).then((allFaces) => {
|
|
||||||
console.log(`...reading ${allFaces.length} descriptors...`);
|
|
||||||
return Promise.map(allFaces, (face) => {
|
|
||||||
const id = face.id,
|
|
||||||
dataPath = faceData + "/" + (id % 100) + "/" + id + "-data.json";
|
|
||||||
|
|
||||||
if (id in descriptors) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
return exists(dataPath).then((doesExist) => {
|
|
||||||
if (!doesExist) {
|
|
||||||
console.warn(`${dataPath} is missing!`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
descriptors[id] = JSON.parse(fs.readFileSync(dataPath));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
const total = facesToUpdate.length;
|
|
||||||
let remaining = total,
|
|
||||||
processed = 0,
|
|
||||||
lastStatus = Date.now(),
|
|
||||||
targets = [];
|
|
||||||
|
|
||||||
for (let target in descriptors) {
|
|
||||||
targets.push({ id: target, descriptor: descriptors[target] });
|
|
||||||
}
|
|
||||||
|
|
||||||
return Promise.mapSeries(facesToUpdate, (face) => {
|
|
||||||
if (!(face.id in descriptors)) {
|
|
||||||
console.warn(`...attempt to compare distance with no descriptor for ${face.id}`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const faceDescriptor = descriptors[face.id];
|
|
||||||
|
|
||||||
return photoDB.sequelize.transaction((transaction) => {
|
|
||||||
return photoDB.sequelize.query(
|
|
||||||
"SELECT distance,face1Id,face2Id " +
|
|
||||||
"FROM facedistances " +
|
|
||||||
"WHERE face1Id=:id OR face2Id=:id " +
|
|
||||||
"ORDER BY face1Id ASC", {
|
|
||||||
replacements: {
|
|
||||||
id: face.id
|
|
||||||
},
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
||||||
raw: true,
|
|
||||||
transaction: transaction
|
|
||||||
}).then((distances) => {
|
|
||||||
return Promise.map(targets, (target) => {
|
|
||||||
/* Skip comparing to self */
|
|
||||||
if (target.id == face.id) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Only compare against newer faces */
|
|
||||||
if (face.lastComparedId && target.id <= face.lastComparedId) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const index = distances.findIndex((distance) => {
|
|
||||||
return distance.face1Id == target.id || distance.face2Id == target.id
|
|
||||||
});
|
|
||||||
|
|
||||||
if (index != -1) {
|
|
||||||
/* A distance has already been calculated between face and target */
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const distance = faceapi.euclideanDistance(faceDescriptor, target.descriptor);
|
|
||||||
|
|
||||||
/* If the distance > 0.6, we don't want to store this in the DB */
|
|
||||||
if (distance > 0.6) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (distance < 0.4) {
|
|
||||||
process.stdout.write(".");
|
|
||||||
// console.log(`Face ${face.id} and ${target.id} have a distance of: ${distance}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return photoDB.sequelize.query(
|
|
||||||
"INSERT INTO facedistances (face1Id,face2Id,distance) " +
|
|
||||||
"VALUES (:first,:second,:distance)", {
|
|
||||||
replacements: {
|
|
||||||
first: Math.min(face.id, target.id),
|
|
||||||
second: Math.max(face.id, target.id),
|
|
||||||
distance: distance
|
|
||||||
},
|
|
||||||
transaction: transaction
|
|
||||||
});
|
|
||||||
}, {
|
|
||||||
concurrency: maxConcurrency
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
return photoDB.sequelize.query(
|
|
||||||
"UPDATE faces SET lastComparedId=:lastId WHERE id=:id", {
|
|
||||||
replacements: {
|
|
||||||
lastId: maxId,
|
|
||||||
id: face.id
|
|
||||||
},
|
|
||||||
transaction: transaction
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
processed++;
|
|
||||||
const now = Date.now();
|
|
||||||
if (now - lastStatus > 5000) {
|
|
||||||
const rate = Math.round(10000 * (remaining - (total - processed)) / (now - lastStatus)) / 10,
|
|
||||||
eta = Math.round((total - processed) / rate);
|
|
||||||
lastStatus = now;
|
|
||||||
remaining = total - processed;
|
|
||||||
console.log(`\nProcessing ${rate} faces per second. ${remaining} faces to be processed. ETA: ${eta}s`);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
console.log("Face detection scanning completed.");
|
|
||||||
}).catch((error) => {
|
|
||||||
console.error(error);
|
|
||||||
process.exit(-1);
|
|
||||||
});
|
|
||||||
@ -1,281 +0,0 @@
|
|||||||
"use strict";
|
|
||||||
|
|
||||||
process.env.TZ = "Etc/GMT";
|
|
||||||
|
|
||||||
require('@tensorflow/tfjs-node');
|
|
||||||
|
|
||||||
let photoDB = null;
|
|
||||||
|
|
||||||
const config = require("config"),
|
|
||||||
Promise = require("bluebird"),
|
|
||||||
{ exists, mkdir, unlink } = require("./lib/util"),
|
|
||||||
faceapi = require("face-api.js"),
|
|
||||||
fs = require("fs"),
|
|
||||||
canvas = require("canvas");
|
|
||||||
|
|
||||||
const { createCanvas, Canvas, Image, ImageData } = canvas;
|
|
||||||
|
|
||||||
faceapi.env.monkeyPatch({ Canvas, Image, ImageData });
|
|
||||||
|
|
||||||
const maxConcurrency = require("os").cpus().length;
|
|
||||||
|
|
||||||
require("./console-line.js"); /* Monkey-patch console.log with line numbers */
|
|
||||||
|
|
||||||
const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/",
|
|
||||||
faceData = picturesPath + "face-data/";
|
|
||||||
|
|
||||||
function alignFromLandmarks(image, landmarks, drawLandmarks) {
|
|
||||||
const faceMargin = 0.45,
|
|
||||||
width = 512, height = 512,
|
|
||||||
dY = landmarks._positions[45]._y - landmarks._positions[36]._y,
|
|
||||||
dX = landmarks._positions[45]._x - landmarks._positions[36]._x,
|
|
||||||
mid = {
|
|
||||||
x: landmarks._positions[36]._x + 0.5 * dX,
|
|
||||||
y: landmarks._positions[36]._y + 0.5 * dY
|
|
||||||
},
|
|
||||||
rotation = -Math.atan2(dY, dX),
|
|
||||||
cosRotation = Math.cos(rotation),
|
|
||||||
sinRotation = Math.sin(rotation),
|
|
||||||
eyeDistance = Math.sqrt(dY * dY + dX * dX),
|
|
||||||
scale = width * (1.0 - 2. * faceMargin) / eyeDistance,
|
|
||||||
canvas = createCanvas(width, height),
|
|
||||||
ctx = canvas.getContext("2d");
|
|
||||||
|
|
||||||
const prime = {
|
|
||||||
x: mid.x * cosRotation - mid.y * sinRotation,
|
|
||||||
y: mid.y * cosRotation + mid.x * sinRotation
|
|
||||||
};
|
|
||||||
|
|
||||||
mid.x = prime.x;
|
|
||||||
mid.y = prime.y;
|
|
||||||
|
|
||||||
ctx.translate(
|
|
||||||
0.5 * width - mid.x * scale,
|
|
||||||
0.5 * height - (height * (0.5 - faceMargin)) - mid.y * scale);
|
|
||||||
ctx.rotate(rotation);
|
|
||||||
ctx.scale(scale, scale);
|
|
||||||
ctx.drawImage(image, 0, 0);
|
|
||||||
|
|
||||||
if (drawLandmarks) {
|
|
||||||
ctx.strokeStyle = "red";
|
|
||||||
ctx.strokeWidth = "1";
|
|
||||||
ctx.beginPath();
|
|
||||||
landmarks._positions.forEach((point, index) => {
|
|
||||||
if (index == 0) {
|
|
||||||
ctx.moveTo(point._x, point._y);
|
|
||||||
} else {
|
|
||||||
ctx.lineTo(point._x, point._y);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
ctx.stroke();
|
|
||||||
}
|
|
||||||
|
|
||||||
return canvas;
|
|
||||||
}
|
|
||||||
|
|
||||||
process.stdout.write("Loading DB.");
|
|
||||||
require("./db/photos").then(function(db) {
|
|
||||||
process.stdout.write("done\n");
|
|
||||||
photoDB = db;
|
|
||||||
}).then(() => {
|
|
||||||
console.log("DB connected.");
|
|
||||||
process.stdout.write("Loading models.");
|
|
||||||
return faceapi.nets.ssdMobilenetv1.loadFromDisk('./models');
|
|
||||||
}).then(() => {
|
|
||||||
process.stdout.write(".");
|
|
||||||
return faceapi.nets.faceLandmark68Net.loadFromDisk('./models');
|
|
||||||
}).then(() => {
|
|
||||||
process.stdout.write(".");
|
|
||||||
return faceapi.nets.faceRecognitionNet.loadFromDisk('./models');
|
|
||||||
}).then(async () => {
|
|
||||||
process.stdout.write(".done\n");
|
|
||||||
|
|
||||||
if (process.argv[0].match(/node/)) {
|
|
||||||
process.argv.shift(); /* node */
|
|
||||||
}
|
|
||||||
process.argv.shift(); /* script name */
|
|
||||||
|
|
||||||
return Promise.resolve().then(() => {
|
|
||||||
if (process.argv.length != 0) {
|
|
||||||
return process.argv;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* If no parameters provided, scan all faces to create image crops */
|
|
||||||
return photoDB.sequelize.query("SELECT id FROM faces ORDER BY id ASC", {
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
||||||
raw: true
|
|
||||||
}).then((results) => {
|
|
||||||
return results.map(result => result.id);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then((args) => {
|
|
||||||
const faces = [];
|
|
||||||
|
|
||||||
console.log(`Scanning ${args.length} faces.`);
|
|
||||||
return Promise.map(args, (arg) => {
|
|
||||||
const file = arg;
|
|
||||||
let id = parseInt(arg);
|
|
||||||
|
|
||||||
let loader;
|
|
||||||
|
|
||||||
if (id == file) {
|
|
||||||
/* This is a face id */
|
|
||||||
console.log(`Looking up face-id ${id}...`);
|
|
||||||
loader = photoDB.sequelize.query(
|
|
||||||
"SELECT albums.path,photos.filename,photos.width,photos.height,faces.* " +
|
|
||||||
"FROM faces,photos,albums " +
|
|
||||||
"WHERE photos.id=faces.photoId " +
|
|
||||||
"AND albums.id=photos.albumId " +
|
|
||||||
"AND faces.id=:id", {
|
|
||||||
replacements: {
|
|
||||||
id: id
|
|
||||||
},
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT,
|
|
||||||
raw: true
|
|
||||||
}).then((results) => {
|
|
||||||
if (results.length != 1) {
|
|
||||||
console.error(`...error. No face-id found: ${id}.\n`);
|
|
||||||
process.exit(-1);
|
|
||||||
}
|
|
||||||
const photo = results[0];
|
|
||||||
console.log(`...loading ${photo.filename}`);
|
|
||||||
|
|
||||||
const file = photo.path + photo.filename;
|
|
||||||
return canvas.loadImage(picturesPath + file).then(async (image) => {
|
|
||||||
const detectors = [ {
|
|
||||||
detection: {
|
|
||||||
_box: {
|
|
||||||
_x: photo.left * photo.width,
|
|
||||||
_y: photo.top * photo.height,
|
|
||||||
_width: (photo.right - photo.left) * photo.width,
|
|
||||||
_height: (photo.bottom - photo.top) * photo.height,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
descriptor: JSON.parse(fs.readFileSync(faceData + (id % 100) + "/" + id + "-data.json"))
|
|
||||||
} ];
|
|
||||||
return [ file, image, detectors ];
|
|
||||||
});
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
/* This is a file */
|
|
||||||
console.log(`Loading ${file}...`);
|
|
||||||
id = undefined;
|
|
||||||
loader = canvas.loadImage(picturesPath + file).then(async (image) => {
|
|
||||||
const detectors = await faceapi.detectAllFaces(image,
|
|
||||||
new faceapi.SsdMobilenetv1Options({
|
|
||||||
minConfidence: 0.9
|
|
||||||
})
|
|
||||||
).withFaceLandmarks();
|
|
||||||
|
|
||||||
await detectors.forEach(async (detector, index) => {
|
|
||||||
const canvas = alignFromLandmarks(image, detector.landmarks, false);
|
|
||||||
fs.writeFileSync(`rotation-pre-${index}.png`, canvas.toBuffer("image/png", {
|
|
||||||
quality: 0.95,
|
|
||||||
chromaSubsampling: false
|
|
||||||
}));
|
|
||||||
const detected = await faceapi.detectSingleFace(canvas,
|
|
||||||
new faceapi.SsdMobilenetv1Options({
|
|
||||||
minConfidence: 0.1
|
|
||||||
})
|
|
||||||
).withFaceLandmarks();
|
|
||||||
const descriptor = await faceapi.computeFaceDescriptor(canvas);
|
|
||||||
console.log(`Processing face ${index}...`);
|
|
||||||
console.log(`...pre aligned score: ${detector.detection._score}`);
|
|
||||||
if (!detected) {
|
|
||||||
console.log("No face found in re-scaled and aligned image");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
console.log(`...post-aligned score: ${detected.detection._score}`);
|
|
||||||
const newCanvas = alignFromLandmarks(canvas, detected.landmarks, true);
|
|
||||||
|
|
||||||
fs.writeFileSync(`rotation-post-${index}.png`, newCanvas.toBuffer("image/png", {
|
|
||||||
quality: 0.95,
|
|
||||||
chromaSubsampling: false
|
|
||||||
}));
|
|
||||||
|
|
||||||
console.log(`Wrote rotation-${index}.png`);
|
|
||||||
|
|
||||||
const data = [];
|
|
||||||
/* Confert from sparse object to dense array */
|
|
||||||
for (let i = 0; i < 128; i++) {
|
|
||||||
data.push(descriptor[i]);
|
|
||||||
}
|
|
||||||
detector.descriptor = data;
|
|
||||||
});
|
|
||||||
|
|
||||||
return [ file, image, detectors ];
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return loader.then((results) => {
|
|
||||||
const filepath = results[0],
|
|
||||||
image = results[1],
|
|
||||||
detectors = results[2];
|
|
||||||
|
|
||||||
process.stdout.write(`${detectors.length} faces.\n`);
|
|
||||||
|
|
||||||
return Promise.map(detectors, (face, index) => {
|
|
||||||
faces.push({
|
|
||||||
filepath: filepath,
|
|
||||||
index: index,
|
|
||||||
descriptor: face.descriptor
|
|
||||||
})
|
|
||||||
|
|
||||||
/* If this is a face-id, output the -original.png
|
|
||||||
* meta-data file */
|
|
||||||
if (!id) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const path = "face-data/" + (id % 100),
|
|
||||||
target = `${path}/${id}-original.png`,
|
|
||||||
box = face.detection._box,
|
|
||||||
aspect = box._width / box._height,
|
|
||||||
dx = (aspect > 1.0) ? 200 : (200 * aspect),
|
|
||||||
dy = (aspect < 1.0) ? 200 : (200 / aspect);
|
|
||||||
|
|
||||||
return exists(target).then((doesExist) => {
|
|
||||||
if (doesExist) {
|
|
||||||
console.log(`...${target} already exists.`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const canvas = createCanvas(200, 200),
|
|
||||||
ctx = canvas.getContext('2d');
|
|
||||||
|
|
||||||
ctx.fillStyle = "rgba(0, 0, 0, 0)";
|
|
||||||
ctx.fillRect(0, 0, 200, 200);
|
|
||||||
ctx.drawImage(image, box._x, box._y, box._width, box._height,
|
|
||||||
Math.floor((200 - dx) * 0.5),
|
|
||||||
Math.floor((200 - dy) * 0.5), dx, dy);
|
|
||||||
|
|
||||||
console.log(`...writing to ${target}.`);
|
|
||||||
|
|
||||||
return mkdir(path).then(() => {
|
|
||||||
fs.writeFileSync(picturesPath + target, canvas.toBuffer("image/png", {
|
|
||||||
quality: 0.95,
|
|
||||||
chromaSubsampling: false
|
|
||||||
}));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}, {
|
|
||||||
concurrency: maxConcurrency
|
|
||||||
}).then(() => {
|
|
||||||
console.log("Face detection scanning completed.");
|
|
||||||
if (0) faces.forEach((a, i) => {
|
|
||||||
faces.forEach((b, j) => {
|
|
||||||
if (i == j) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const distance = faceapi.euclideanDistance(a.descriptor, b.descriptor);
|
|
||||||
if (distance < 0.4) {
|
|
||||||
console.log(`${a.filepath}.${a.index} is similar to ${b.filepath}.${b.index}: ${distance}`);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).catch((error) => {
|
|
||||||
console.error(error);
|
|
||||||
process.exit(-1);
|
|
||||||
});
|
|
||||||
@ -1,88 +0,0 @@
|
|||||||
"use strict";
|
|
||||||
|
|
||||||
const config = require("config"),
|
|
||||||
fs = require("fs"),
|
|
||||||
Promise = require("bluebird"),
|
|
||||||
picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
|
|
||||||
|
|
||||||
const stat = function (_path) {
|
|
||||||
if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) {
|
|
||||||
_path = _path.substring(picturesPath.length);
|
|
||||||
}
|
|
||||||
|
|
||||||
let path = picturesPath + _path;
|
|
||||||
|
|
||||||
return new Promise(function (resolve, reject) {
|
|
||||||
fs.stat(path, function (error, stats) {
|
|
||||||
if (error) {
|
|
||||||
return reject(error);
|
|
||||||
}
|
|
||||||
return resolve(stats);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const unlink = function (_path) {
|
|
||||||
if (_path.indexOf(picturesPath.replace(/\/$/, "")) == 0) {
|
|
||||||
_path = _path.substring(picturesPath.length);
|
|
||||||
}
|
|
||||||
|
|
||||||
let path = picturesPath + _path;
|
|
||||||
|
|
||||||
return new Promise(function (resolve, reject) {
|
|
||||||
fs.unlink(path, function (error) {
|
|
||||||
if (error) {
|
|
||||||
return reject(error);
|
|
||||||
}
|
|
||||||
return resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const mkdir = function (_path) {
|
|
||||||
if (_path.indexOf(picturesPath) == 0) {
|
|
||||||
_path = _path.substring(picturesPath.length);
|
|
||||||
}
|
|
||||||
|
|
||||||
let parts = _path.split("/"), path;
|
|
||||||
|
|
||||||
parts.unshift(picturesPath);
|
|
||||||
return Promise.mapSeries(parts, function (part) {
|
|
||||||
if (!path) {
|
|
||||||
path = picturesPath.replace(/\/$/, "");
|
|
||||||
} else {
|
|
||||||
path += "/" + part;
|
|
||||||
}
|
|
||||||
|
|
||||||
return stat(path).catch(function (error) {
|
|
||||||
if (error.code != "ENOENT") {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Promise(function (resolve, reject) {
|
|
||||||
fs.mkdir(path, function (error) {
|
|
||||||
if (error) {
|
|
||||||
return reject(error);
|
|
||||||
}
|
|
||||||
|
|
||||||
return resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const exists = function(path) {
|
|
||||||
return stat(path).then(function() {
|
|
||||||
return true;
|
|
||||||
}).catch(function() {
|
|
||||||
return false;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
stat,
|
|
||||||
exists,
|
|
||||||
mkdir,
|
|
||||||
unlink
|
|
||||||
};
|
|
||||||
@ -1,748 +0,0 @@
|
|||||||
/**
|
|
||||||
* scanner
|
|
||||||
*
|
|
||||||
* Face recognition:
|
|
||||||
* 1. For each photo, extract all faces. Store face rectangles.
|
|
||||||
* face_id unique
|
|
||||||
* photo_id foreign key
|
|
||||||
* top left bottom right
|
|
||||||
* identity_id
|
|
||||||
* distance (0 == truth; manually assigned identity)
|
|
||||||
* 2. For each face_id, create:
|
|
||||||
* normalized_file
|
|
||||||
* original_file
|
|
||||||
* 128 float
|
|
||||||
*/
|
|
||||||
"use strict";
|
|
||||||
|
|
||||||
/* meta directories are not scanned for photos */
|
|
||||||
const metaDirectories = [ "thumbs", "raw", "face-data", ".git", "corrupt" ];
|
|
||||||
|
|
||||||
const Promise = require("bluebird"),
|
|
||||||
fs = require("fs"),
|
|
||||||
config = require("config"),
|
|
||||||
moment = require("moment"),
|
|
||||||
crypto = require("crypto"),
|
|
||||||
{ stat, mkdir, exists } = require("./lib/util");
|
|
||||||
|
|
||||||
let photoDB = null;
|
|
||||||
|
|
||||||
const picturesPath = config.get("picturesPath").replace(/\/$/, "") + "/";
|
|
||||||
|
|
||||||
let processQueue = [], triedClean = [], lastScan = new Date("1800-01-01");
|
|
||||||
|
|
||||||
//const rawExtension = /\.(nef|orf)$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef", "orf" ];
|
|
||||||
|
|
||||||
const rawExtension = /\.nef$/i, extensions = [ "jpg", "jpeg", "png", "gif", "nef" ];
|
|
||||||
|
|
||||||
function removeNewerFile(path, fileA, fileB) {
|
|
||||||
fs.stat(path + fileA, function(err, statsA) {
|
|
||||||
if (err) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
fs.stat(path + fileB, function(err, statsB) {
|
|
||||||
if (err) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (statsA.mtime > statsB.mtime) {
|
|
||||||
setStatus("Removing file by moving to 'corrupt':" + fileA);
|
|
||||||
moveCorrupt(path, fileA);
|
|
||||||
} else {
|
|
||||||
setStatus("Removing file by moving to 'corrupt':" + fileB);
|
|
||||||
moveCorrupt(path, fileB);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let processRunning = false;
|
|
||||||
|
|
||||||
const { spawn } = require('child_process');
|
|
||||||
|
|
||||||
const sharp = require("sharp"), exif = require("exif-reader");
|
|
||||||
|
|
||||||
function convertRawToJpg(path, raw, file) {
|
|
||||||
setStatus(`Converting ${path}${raw} to ${file}.`);
|
|
||||||
|
|
||||||
path = picturesPath + path;
|
|
||||||
|
|
||||||
return new Promise(function(resolve, reject) {
|
|
||||||
return exists(path + file.replace(rawExtension, ".jpg")).then(function(exist) {
|
|
||||||
if (exist) {
|
|
||||||
setStatus("Skipping already converted file: " + file);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const darktable = spawn("darktable-cli", [
|
|
||||||
path + raw,
|
|
||||||
path + file
|
|
||||||
]);
|
|
||||||
|
|
||||||
const stderr = [];
|
|
||||||
darktable.stderr.on('data', function(data) {
|
|
||||||
stderr.push(data);
|
|
||||||
});
|
|
||||||
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
darktable.on('exit', (code, signal) => {
|
|
||||||
if (signal || code != 0) {
|
|
||||||
let error = "darktable for " + path + file + " returned an error: " + code + "\n" + signal + "\n" + stderr.join("\n") + "\n";
|
|
||||||
setStatus(error, "error");
|
|
||||||
return moveCorrupt(path, file).then(function() {
|
|
||||||
setStatus("darktable failed", "warn");
|
|
||||||
return reject(error);
|
|
||||||
}).catch(function(error) {
|
|
||||||
setStatus("moveCorrupt failed", "warn");
|
|
||||||
return reject(error);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return mkdir(path + "raw").then(function() {
|
|
||||||
fs.rename(path + raw, path + "raw/" + raw, function(err) {
|
|
||||||
if (err) {
|
|
||||||
setStatus("Unable to move RAW file: " + path + raw, "error");
|
|
||||||
return reject(err);
|
|
||||||
}
|
|
||||||
return resolve();
|
|
||||||
});
|
|
||||||
}).catch(function(error) {
|
|
||||||
setStatus("mkdir failed", "warn");
|
|
||||||
return reject(error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function moveCorrupt(path, file) {
|
|
||||||
if (path.indexOf(picturesPath) != 0) {
|
|
||||||
path = picturesPath + path;
|
|
||||||
}
|
|
||||||
|
|
||||||
setStatus("Moving corrupt file '" + file + "' to " + path + "corrupt", "warn");
|
|
||||||
|
|
||||||
return mkdir(path + "corrupt").then(function() {
|
|
||||||
return new Promise(function(resolve, reject) {
|
|
||||||
fs.rename(path + file, path + "corrupt/" + file, function(err) {
|
|
||||||
if (err) {
|
|
||||||
setStatus("Unable to move corrupt file: " + path + file, "error");
|
|
||||||
return reject(err);
|
|
||||||
}
|
|
||||||
return resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function processBlock(items) {
|
|
||||||
|
|
||||||
if (items) {
|
|
||||||
processQueue = processQueue.concat(items);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (processRunning) {
|
|
||||||
/* Invoke once per second to check if there are items to process */
|
|
||||||
setTimeout(processBlock, 1000);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let processing = processQueue.splice(0), needsProcessing = [], duplicates = [];
|
|
||||||
|
|
||||||
processRunning = true;
|
|
||||||
|
|
||||||
/* Sort to newest files to be processed first */
|
|
||||||
processing.sort(function(a, b) {
|
|
||||||
return b.stats.mtime - a.stats.mtime;
|
|
||||||
});
|
|
||||||
|
|
||||||
let toProcess = processing.length, lastMessage = moment();
|
|
||||||
setStatus("Items to be processed: " + toProcess);
|
|
||||||
return Promise.mapSeries(processing, (asset) => {
|
|
||||||
if (!asset.raw) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const path = asset.album.path;
|
|
||||||
|
|
||||||
return exists(picturesPath + path + asset.filename).then(function(exist) {
|
|
||||||
if (exist) {
|
|
||||||
return asset;
|
|
||||||
}
|
|
||||||
|
|
||||||
return mkdir(picturesPath + path + "raw").then(function() {
|
|
||||||
return convertRawToJpg(path, asset.raw, asset.filename);
|
|
||||||
}).then(function() {
|
|
||||||
console.log("Done converting...");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(() => {
|
|
||||||
return Promise.mapSeries(processing, (asset) => {
|
|
||||||
return computeHash(picturesPath + asset.album.path + asset.filename).then(function(hash) {
|
|
||||||
asset.hash = hash;
|
|
||||||
return asset;
|
|
||||||
}).then(function(asset) {
|
|
||||||
return photoDB.sequelize.query("SELECT photohashes.*,photos.filename,albums.path FROM photohashes " +
|
|
||||||
"LEFT JOIN photos ON (photos.id=photohashes.photoId) " +
|
|
||||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
|
||||||
"WHERE hash=:hash OR photoId=:id", {
|
|
||||||
replacements: asset,
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT
|
|
||||||
}).then(function(results) {
|
|
||||||
let query;
|
|
||||||
|
|
||||||
if (results.length == 0) {
|
|
||||||
query = "INSERT INTO photohashes (hash,photoId) VALUES(:hash,:id)";
|
|
||||||
} else if (results[0].hash != asset.hash) {
|
|
||||||
query = "UPDATE photohashes SET hash=:hash WHERE photoId=:id";
|
|
||||||
} else if (results[0].photoId != asset.id) {
|
|
||||||
setStatus("Duplicate asset: " +
|
|
||||||
"'" + asset.album.path + asset.filename + "' is a copy of " +
|
|
||||||
"'" + results[0].path + results[0].filename + "'");
|
|
||||||
if (asset.duplicate != results[0].photoId) {
|
|
||||||
asset.duplicate = results[0].photoId;
|
|
||||||
duplicates.push(asset);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Even if the hash doesn't need to be updated, the entry needs to be scanned */
|
|
||||||
// console.log("process needed because of " + query);
|
|
||||||
needsProcessing.push(asset);
|
|
||||||
|
|
||||||
if (!query) {
|
|
||||||
return asset;
|
|
||||||
}
|
|
||||||
|
|
||||||
return photoDB.sequelize.query(query, {
|
|
||||||
replacements: asset,
|
|
||||||
}).then(function() {
|
|
||||||
return asset;
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(function(asset) {
|
|
||||||
if (!asset) { /* The processed entry is a DUPLICATE. Skip it. */
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
var path = asset.album.path,
|
|
||||||
file = asset.filename,
|
|
||||||
created = asset.stats.mtime,
|
|
||||||
albumId = asset.album.id;
|
|
||||||
|
|
||||||
var src = picturesPath + path + file,
|
|
||||||
image = sharp(src);
|
|
||||||
|
|
||||||
return image.limitInputPixels(1073741824).metadata().then(function(metadata) {
|
|
||||||
if (metadata.exif) {
|
|
||||||
metadata.exif = exif(metadata.exif);
|
|
||||||
delete metadata.exif.thumbnail;
|
|
||||||
delete metadata.exif.image;
|
|
||||||
for (var key in metadata.exif.exif) {
|
|
||||||
if (Buffer.isBuffer(metadata.exif.exif[key])) {
|
|
||||||
metadata.exif.exif[key] = "Buffer[" + metadata.exif.exif[key].length + "]";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
asset.width = metadata.width;
|
|
||||||
asset.height = metadata.height;
|
|
||||||
asset.added = moment().format();
|
|
||||||
|
|
||||||
if (metadata.exif && metadata.exif.exif && metadata.exif.exif.DateTimeOriginal && !isNaN(metadata.exif.exif.DateTimeOriginal.valueOf())) {
|
|
||||||
asset.taken = moment(metadata.exif.exif.DateTimeOriginal).format();
|
|
||||||
asset.modified = moment(metadata.exif.exif.DateTimeOriginal).format();
|
|
||||||
|
|
||||||
if (asset.taken == "Invalid date" || asset.taken.replace(/T.*/, "") == "1899-11-30") {
|
|
||||||
setStatus("Invalid EXIF date information for " + asset.album.path + asset.filename);
|
|
||||||
asset.taken = asset.modified = moment(created).format();
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
/* Attempt to infer the datestamp from the filename */
|
|
||||||
let date = moment(created).format();
|
|
||||||
|
|
||||||
let match = file.match(/WhatsApp Image (20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]) at (.*).(jpeg|jpg)/);
|
|
||||||
if (match) {
|
|
||||||
date = moment((match[1]+" "+match[2]), "YYYY-MM-DD h.mm.ss a").format();
|
|
||||||
if (date == "Invalid date") {
|
|
||||||
date = moment(created).format();
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
match = file.match(/(20[0-9][0-9]-?[0-9][0-9]-?[0-9][0-9])[_\-]?([0-9]{6})?/);
|
|
||||||
if (match) {
|
|
||||||
if (match[2]) { /* Stamp had time in it */
|
|
||||||
date = moment((match[1]+""+match[2]).replace(/-/g, ""), "YYYYMMDDHHmmss").format();
|
|
||||||
} else {
|
|
||||||
date = moment(match[1].replace(/-/g, ""), "YYYYMMDD").format();
|
|
||||||
}
|
|
||||||
if (date == "Invalid date") {
|
|
||||||
date = moment(created).format();
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
date = moment(created).format();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
asset.taken = asset.modified = date;
|
|
||||||
}
|
|
||||||
|
|
||||||
let dst = picturesPath + path + "thumbs/" + file;
|
|
||||||
|
|
||||||
return exists(dst).then(function(exist) {
|
|
||||||
if (exist) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
return image.resize(256, 256).withMetadata().toFile(dst).catch(function(error) {
|
|
||||||
setStatus("Error resizing image: " + dst + "\n" + error, "error");
|
|
||||||
throw error;
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
let dst = picturesPath + path + "thumbs/scaled/" + file;
|
|
||||||
return exists(dst).then(function(exist) {
|
|
||||||
if (exist) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
return image.resize(Math.min(1024, metadata.width)).withMetadata().toFile(dst).catch(function(error) {
|
|
||||||
setStatus("Error resizing image: " + dst + "\n" + error, "error");
|
|
||||||
throw error;
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
return photoDB.sequelize.query("UPDATE photos SET " +
|
|
||||||
"added=:added,modified=:modified,taken=:taken,width=:width,height=:height,size=:size,scanned=CURRENT_TIMESTAMP " +
|
|
||||||
"WHERE id=:id", {
|
|
||||||
replacements: asset,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).catch(function(error) {
|
|
||||||
setStatus("Error reading image " + src + ":\n" + error, "error");
|
|
||||||
return moveCorrupt(path, file);
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
toProcess--;
|
|
||||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
|
||||||
setStatus("Items to be processed: " + toProcess);
|
|
||||||
lastMessage = moment();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).catch(function(error) {
|
|
||||||
setStatus("Error processing file. Continuing.", "error");
|
|
||||||
throw error;
|
|
||||||
}).then(function() {
|
|
||||||
setStatus("Completed processing queue. Marking " + duplicates.length + " duplicates.");
|
|
||||||
return photoDB.sequelize.transaction(function(transaction) {
|
|
||||||
return Promise.mapSeries(duplicates, function(asset) {
|
|
||||||
return photoDB.sequelize.query("UPDATE photos " +
|
|
||||||
"SET duplicate=:duplicate,modified=CURRENT_TIMESTAMP,scanned=CURRENT_TIMESTAMP WHERE id=:id", {
|
|
||||||
replacements: asset,
|
|
||||||
transaction: transaction
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
setStatus("Looking for removed assets");
|
|
||||||
return photoDB.sequelize.query("SELECT photos.scanned,photos.id,photos.filename,albums.path FROM photos " +
|
|
||||||
"LEFT JOIN albums ON (albums.id=photos.albumId) " +
|
|
||||||
"WHERE photos.deleted=0 AND (DATETIME(photos.scanned)<DATETIME(:lastScan) OR photos.scanned IS NULL)", {
|
|
||||||
replacements: {
|
|
||||||
lastScan: lastScan
|
|
||||||
},
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT
|
|
||||||
}).then(function(results) {
|
|
||||||
let deleted = [];
|
|
||||||
setStatus("Checking " + results.length + " assets to see if they are on disk.");
|
|
||||||
return Promise.map(results, function(asset) {
|
|
||||||
return exists(asset.path + asset.filename).then(function(exist) {
|
|
||||||
if (!exist) {
|
|
||||||
setStatus(asset.path + asset.filename + " no longer exists on disk. Marking as deleted.");
|
|
||||||
deleted.push(asset.id);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
return photoDB.sequelize.query("UPDATE photos SET deleted=1,scanned=CURRENT_TIMESTAMP WHERE id IN (:deleted)", {
|
|
||||||
replacements: {
|
|
||||||
deleted: deleted
|
|
||||||
}
|
|
||||||
}).then(function() {
|
|
||||||
return photoDB.sequelize.query("DELETE FROM photohashes WHERE photoId IN (:deleted)", {
|
|
||||||
replacements: {
|
|
||||||
deleted: deleted
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
setStatus(deleted.length + " assets deleted.");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
processRunning = false;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function scanDir(parent, path) {
|
|
||||||
let re = new RegExp("\.((" + extensions.join(")|(") + "))$", "i"),
|
|
||||||
album = {
|
|
||||||
path: path.slice(picturesPath.length), /* path already ends in '/' */
|
|
||||||
name: path.replace(/\/$/, "").replace(/.*\//, "").replace(/_/g, " "),
|
|
||||||
parent: parent,
|
|
||||||
allAssetCount: 0,
|
|
||||||
allAlbumCount: 0
|
|
||||||
}, albums = [ album ], assets = [];
|
|
||||||
|
|
||||||
return new Promise(function(resolve, reject) {
|
|
||||||
fs.readdir(path, function(error, files) {
|
|
||||||
if (error) {
|
|
||||||
setStatus("Could not readdir: " + path, "warn");
|
|
||||||
return resolve([]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Remove meta-data directories from being processed */
|
|
||||||
files = files.filter((file) => {
|
|
||||||
for (var i = 0; i < files.length; i++) {
|
|
||||||
/* If this file has an original NEF/ORF on the system, don't add the JPG to the DB */
|
|
||||||
if (rawExtension.exec(files[i]) && file == files[i].replace(rawExtension, ".jpg")) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* If there is a different CASE (eg. JPG vs jpg) don't add it, and remove the 'lower case'
|
|
||||||
* version from disk. */
|
|
||||||
if (file != files[i] && file.toUpperCase() == files[i]) {
|
|
||||||
removeNewerFile(path, file, files[i]);
|
|
||||||
setStatus("Duplicate file in " + path + ": ", file, files[i]);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return metaDirectories.indexOf(file) == -1;
|
|
||||||
});
|
|
||||||
|
|
||||||
return resolve(files);
|
|
||||||
});
|
|
||||||
}).then(function(files) {
|
|
||||||
return Promise.map(files, function(file) {
|
|
||||||
let filepath = path + file;
|
|
||||||
return stat(filepath).then(function(stats) {
|
|
||||||
if (stats.isDirectory()) {
|
|
||||||
filepath += "/";
|
|
||||||
return scanDir(album, filepath).spread(function(_albums, _assets) {
|
|
||||||
album.allAssetCount += _assets.length;
|
|
||||||
album.allAlbumCount += _albums.length + 1;
|
|
||||||
albums = albums.concat(_albums);
|
|
||||||
assets = assets.concat(_assets);
|
|
||||||
}).catch(function(error) {
|
|
||||||
setStatus("Could not scanDir " + filepath + ": " + error, "error");
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Check file extensions */
|
|
||||||
if (!re.exec(file)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
album.hasAssets = true;
|
|
||||||
|
|
||||||
const asset = {
|
|
||||||
filename: file.replace(rawExtension, ".jpg"),
|
|
||||||
name: file.replace(/.[^.]*$/, ""),
|
|
||||||
stats: {
|
|
||||||
mtime: stats.mtime,
|
|
||||||
ctime: stats.ctime
|
|
||||||
},
|
|
||||||
size: stats.size,
|
|
||||||
album: album
|
|
||||||
}
|
|
||||||
if (file != asset.filename) {
|
|
||||||
asset.raw = file;
|
|
||||||
}
|
|
||||||
assets.push(asset);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
return Promise.map(albums, function(album) {
|
|
||||||
if (album.hasAssets) {
|
|
||||||
return mkdir(album.path + "thumbs/scaled");
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
return [ albums, assets ];
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function findOrCreateDBAlbum(transaction, album) {
|
|
||||||
let query = "SELECT id FROM albums WHERE path=:path AND ";
|
|
||||||
if (!album.parent) {
|
|
||||||
query += "parentId IS NULL";
|
|
||||||
album.parentId = null;
|
|
||||||
} else {
|
|
||||||
if (!album.parent.id) {
|
|
||||||
let error = "Albums in array in non ancestral order!";
|
|
||||||
setStatus(error, "error");
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
album.parentId = album.parent.id;
|
|
||||||
query += "parentId=:parentId";
|
|
||||||
}
|
|
||||||
|
|
||||||
return photoDB.sequelize.query(query, {
|
|
||||||
replacements: album,
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT
|
|
||||||
}).then(function(results) {
|
|
||||||
if (results.length == 0) {
|
|
||||||
if (!album.parent) {
|
|
||||||
setStatus("Creating top level album: " + picturesPath, "warn" );
|
|
||||||
}
|
|
||||||
return photoDB.sequelize.query("INSERT INTO albums (path,parentId,name) VALUES(:path,:parentId,:name)", {
|
|
||||||
replacements: album,
|
|
||||||
transaction: transaction
|
|
||||||
}).spread(function(results, metadata) {
|
|
||||||
return metadata.lastID;
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
return results[0].id;
|
|
||||||
}
|
|
||||||
}).then(function(id) {
|
|
||||||
album.id = id;
|
|
||||||
return id;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function findOrUpdateDBAsset(transaction, asset) {
|
|
||||||
if (!asset.album || !asset.album.id) {
|
|
||||||
let error = "Asset being processed without an album";
|
|
||||||
setStatus(error, "warn");
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
asset.albumId = asset.album.id;
|
|
||||||
|
|
||||||
return photoDB.sequelize.query(
|
|
||||||
"SELECT id,DATETIME(scanned) AS scanned,size,DATETIME(modified) AS modified " +
|
|
||||||
"FROM photos " +
|
|
||||||
"WHERE albumId=:albumId AND filename=:filename", {
|
|
||||||
replacements: asset,
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT
|
|
||||||
}).then(function(results) {
|
|
||||||
if (results.length == 0) {
|
|
||||||
return photoDB.sequelize.query("INSERT INTO photos " +
|
|
||||||
"(albumId,filename,name,size) VALUES(:albumId,:filename,:name,:size)", {
|
|
||||||
replacements: asset,
|
|
||||||
transaction: transaction
|
|
||||||
}).spread(function(results, metadata) {
|
|
||||||
asset.id = metadata.lastID;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
asset.id = results[0].id;
|
|
||||||
asset.scanned = new Date(results[0].scanned);
|
|
||||||
asset.modified = new Date(results[0].modified);
|
|
||||||
|
|
||||||
/* If the size on disk changed, update the size entry in the DB. This shouldn't happen in
|
|
||||||
* production unless someone modifies the file, then re-stamps the modified time */
|
|
||||||
if (asset.size != results[0].size) {
|
|
||||||
setStatus("File was modified with time-restamp (HASH regeneration will be queued): " + asset.filename);
|
|
||||||
delete asset.scanned;
|
|
||||||
delete asset.modified;
|
|
||||||
}
|
|
||||||
}).then(function() {
|
|
||||||
return asset;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function computeHash(filepath) {
|
|
||||||
return new Promise(function(resolve, reject) {
|
|
||||||
let input = fs.createReadStream(filepath),
|
|
||||||
hash = crypto.createHash("sha256");
|
|
||||||
|
|
||||||
if (!input) {
|
|
||||||
console.warn("Unable to open " + filepath);
|
|
||||||
return reject();
|
|
||||||
}
|
|
||||||
|
|
||||||
input.on("error", function(error) {
|
|
||||||
console.warn("Error reading " + filepath);
|
|
||||||
reject(error);
|
|
||||||
});
|
|
||||||
|
|
||||||
input.on("readable", function() {
|
|
||||||
const data = input.read();
|
|
||||||
if (data) {
|
|
||||||
hash.update(data);
|
|
||||||
} else {
|
|
||||||
input.close();
|
|
||||||
resolve(hash.digest("hex"));
|
|
||||||
hash = null;
|
|
||||||
input = null;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let scanningStatus = [];
|
|
||||||
|
|
||||||
function setStatus(status, level) {
|
|
||||||
if (status == "idle") {
|
|
||||||
scanningStatus = [];
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
level = level || "info";
|
|
||||||
scanningStatus.push({
|
|
||||||
level: level,
|
|
||||||
time: moment().format(),
|
|
||||||
log: status
|
|
||||||
});
|
|
||||||
switch (level) {
|
|
||||||
case "error":
|
|
||||||
console.error(status);
|
|
||||||
break;
|
|
||||||
case "warn":
|
|
||||||
console.warn(status);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
console.log(status);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function doScan() {
|
|
||||||
/* 1. Scan for all assets which will be managed by the system. readdir
|
|
||||||
* 2. Check if entry in DB. Check mod-time in DB vs. stats from #1
|
|
||||||
* - For albums
|
|
||||||
* - For assets
|
|
||||||
* 3. If not in DB, or mod-time changed, queue for HASH CHECK
|
|
||||||
*
|
|
||||||
* HASH CHECK
|
|
||||||
* 1. Compute HASH
|
|
||||||
* 2. Check for HASH in photohash -- skip?
|
|
||||||
* 3. Check for and create thumbs/FILE thumbs/scaled/FILE
|
|
||||||
* 4. If necessary, create JPG from RAW
|
|
||||||
* 5. Update last-scanned date in DB for entry
|
|
||||||
* 6. Look up all DB entries with last-scanned date < NOW -- purge from DB (they were
|
|
||||||
* removed on disk)? Also purge from the HASH table.
|
|
||||||
*/
|
|
||||||
let initialized = Date.now();
|
|
||||||
let now = Date.now();
|
|
||||||
let needsProcessing = [];
|
|
||||||
|
|
||||||
if (scanningStatus.length != 0) {
|
|
||||||
return Promise.resolve(scanningStatus);
|
|
||||||
}
|
|
||||||
|
|
||||||
return scanDir(null, picturesPath).spread(function(albums, assets) {
|
|
||||||
setStatus("Found " + assets.length + " assets in " + albums.length + " albums after " +
|
|
||||||
((Date.now() - now) / 1000) + "s");
|
|
||||||
/* One at a time, in series, as the album[] array has parents first, then descendants.
|
|
||||||
* Operating in parallel could result in a child being searched for prior to the parent */
|
|
||||||
now = Date.now();
|
|
||||||
|
|
||||||
let toProcess = albums.length, lastMessage = moment();
|
|
||||||
return photoDB.sequelize.transaction(function(transaction) {
|
|
||||||
return Promise.mapSeries(albums, function(album) {
|
|
||||||
return findOrCreateDBAlbum(transaction, album).then(function() {
|
|
||||||
toProcess--;
|
|
||||||
if (moment().add(-5, 'seconds') > lastMessage) {
|
|
||||||
setStatus("Albums to be created in DB: " + toProcess);
|
|
||||||
lastMessage = moment();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
setStatus("Processed " + albums.length + " album DB entries in " +
|
|
||||||
((Date.now() - now) / 1000) + "s");
|
|
||||||
now = Date.now();
|
|
||||||
|
|
||||||
setStatus(assets.length + " assets remaining to be verified/updated. ETA N/A");
|
|
||||||
|
|
||||||
let processed = 0, start = Date.now(), last = 0, updateScanned = [], newEntries = 0;
|
|
||||||
return photoDB.sequelize.transaction(function(transaction) {
|
|
||||||
return Promise.map(assets, function(asset) {
|
|
||||||
return Promise.resolve(asset).then(function(asset) {
|
|
||||||
/* If both mtime and ctime of the asset are older than the lastScan, skip it
|
|
||||||
*
|
|
||||||
* Can only do this after a full scan has occurred */
|
|
||||||
if (lastScan != null && asset.stats.mtime < lastScan && asset.stats.ctime < lastScan) {
|
|
||||||
return asset;
|
|
||||||
}
|
|
||||||
|
|
||||||
return findOrUpdateDBAsset(transaction, asset).then(function(asset) {
|
|
||||||
if (!asset.scanned) {
|
|
||||||
newEntries++;
|
|
||||||
}
|
|
||||||
if (!asset.scanned || asset.scanned < asset.stats.mtime || !asset.modified) {
|
|
||||||
// if (!asset.scanned) { console.log("no scan date on asset"); }
|
|
||||||
// if (asset.scanned < asset.stats.mtime) { console.log("scan date older than mtime"); }
|
|
||||||
// if (!asset.modified) { console.log("no mtime."); }
|
|
||||||
needsProcessing.push(asset);
|
|
||||||
} else {
|
|
||||||
updateScanned.push(asset.id);
|
|
||||||
}
|
|
||||||
return asset;
|
|
||||||
}).then(function(asset) {
|
|
||||||
return asset;
|
|
||||||
});
|
|
||||||
}).then(function(asset) {
|
|
||||||
processed++;
|
|
||||||
|
|
||||||
let elapsed = Date.now() - start;
|
|
||||||
if (elapsed < 5000) {
|
|
||||||
return asset;
|
|
||||||
}
|
|
||||||
|
|
||||||
let remaining = assets.length - processed, eta = Math.ceil((elapsed / 1000) * remaining / (processed - last));
|
|
||||||
setStatus(remaining + " assets remaining be verified/updated " +
|
|
||||||
"(" + newEntries + " new entries, " + needsProcessing.length + " need processing," + (processed - newEntries) + " up-to-date so far). ETA " + eta + "s");
|
|
||||||
last = processed;
|
|
||||||
start = Date.now();
|
|
||||||
});
|
|
||||||
}, {
|
|
||||||
concurrency: 10
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
if (updateScanned.length) {
|
|
||||||
return photoDB.sequelize.query("UPDATE photos SET scanned=CURRENT_TIMESTAMP WHERE id IN (:ids)", {
|
|
||||||
replacements: {
|
|
||||||
ids: updateScanned
|
|
||||||
}
|
|
||||||
}).then(function() {
|
|
||||||
setStatus("Updated scan date of " + updateScanned.length + " assets");
|
|
||||||
updateScanned = [];
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}).then(function() {
|
|
||||||
setStatus(newEntries + " assets are new. " + (needsProcessing.length - newEntries) + " assets have been modified.\n" +
|
|
||||||
needsProcessing.length + " assets need HASH computed. " + (assets.length - needsProcessing.length) + " need no update.");;
|
|
||||||
processBlock(needsProcessing);
|
|
||||||
needsProcessing = [];
|
|
||||||
}).then(function() {
|
|
||||||
setStatus("Scanned " + assets.length + " asset DB entries in " +
|
|
||||||
((Date.now() - now) / 1000) + "s");
|
|
||||||
assets = [];
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
setStatus("Total time to initialize DB and all scans: " + ((Date.now() - initialized) / 1000) + "s");
|
|
||||||
return photoDB.sequelize.query("SELECT max(scanned) AS scanned FROM photos", {
|
|
||||||
type: photoDB.sequelize.QueryTypes.SELECT
|
|
||||||
}).then(function(results) {
|
|
||||||
if (results[0].scanned == null) {
|
|
||||||
lastScan = new Date("1800-01-01");
|
|
||||||
} else {
|
|
||||||
lastScan = new Date(results[0].scanned);
|
|
||||||
}
|
|
||||||
setStatus("Updating any asset newer than " + moment(lastScan).format());
|
|
||||||
});
|
|
||||||
}).then(function() {
|
|
||||||
setStatus("idle");
|
|
||||||
return "scan complete";
|
|
||||||
}).catch(function(error) {
|
|
||||||
setStatus(error);
|
|
||||||
throw error;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
init: function(db) {
|
|
||||||
photoDB = db;
|
|
||||||
},
|
|
||||||
scan: doScan
|
|
||||||
};
|
|
||||||
|
|
||||||
18
package.json
18
package.json
@ -4,10 +4,12 @@
|
|||||||
"description": "Settlers",
|
"description": "Settlers",
|
||||||
"main": "index.html",
|
"main": "index.html",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
"start": "NODE_CONFIG_ENV='devel' node server/app.js",
|
||||||
|
"backend": "NODE_CONFIG_ENV='production' node server/app.js"
|
||||||
"start": "webpack-dev-server --mode development --host 0.0.0.0 --config webpack.dev.js",
|
"start": "webpack-dev-server --mode development --host 0.0.0.0 --config webpack.dev.js",
|
||||||
"build": "webpack --config webpack.prod.js",
|
"build": "webpack --config webpack.prod.js",
|
||||||
"watch": "webpack --config webpack.prod.js --watch",
|
"watch": "webpack --config webpack.prod.js --watch",
|
||||||
"backend": "NODE_CONFIG_ENV='production' node ketr.ketran/server/app.js"
|
"backend": "NODE_CONFIG_ENV='production' node server/app.js"
|
||||||
},
|
},
|
||||||
"repository": "ssh://git@gitlab.ketrenos.com/jketreno/ketr.settlers.git",
|
"repository": "ssh://git@gitlab.ketrenos.com/jketreno/ketr.settlers.git",
|
||||||
"author": "James Ketrenos <james_settlers@ketrenos.com>",
|
"author": "James Ketrenos <james_settlers@ketrenos.com>",
|
||||||
@ -33,6 +35,20 @@
|
|||||||
"react-router-dom": "^5.0.1",
|
"react-router-dom": "^5.0.1",
|
||||||
"react-scroll": "^1.7.14",
|
"react-scroll": "^1.7.14",
|
||||||
"react-syntax-highlighter": "^11.0.2"
|
"react-syntax-highlighter": "^11.0.2"
|
||||||
|
"bluebird": "^3.5.5",
|
||||||
|
"config": "^3.1.0",
|
||||||
|
"connect-sqlite3": "^0.9.11",
|
||||||
|
"cookie-parser": "^1.4.4",
|
||||||
|
"express": "^4.17.1",
|
||||||
|
"express-session": "^1.17.1",
|
||||||
|
"handlebars": "^4.7.6",
|
||||||
|
"moment": "^2.24.0",
|
||||||
|
"morgan": "^1.9.1",
|
||||||
|
"node-fetch": "^2.6.0",
|
||||||
|
"node-gzip": "^1.1.2",
|
||||||
|
"nodemailer": "^6.3.0",
|
||||||
|
"sequelize": "^5.21.6",
|
||||||
|
"sqlite3": "^4.1.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@babel/cli": "^7.1.0",
|
"@babel/cli": "^7.1.0",
|
||||||
|
|||||||
@ -15,7 +15,8 @@ const express = require("express"),
|
|||||||
|
|
||||||
require("./console-line.js"); /* Monkey-patch console.log with line numbers */
|
require("./console-line.js"); /* Monkey-patch console.log with line numbers */
|
||||||
|
|
||||||
const serverConfig = config.get("server");
|
const frontEndPath = config.get("frontendPath").replace(/\/$/, "") + "/",
|
||||||
|
serverConfig = config.get("server");
|
||||||
|
|
||||||
let basePath = config.get("basePath");
|
let basePath = config.get("basePath");
|
||||||
basePath = "/" + basePath.replace(/^\/+/, "").replace(/\/+$/, "") + "/";
|
basePath = "/" + basePath.replace(/^\/+/, "").replace(/\/+$/, "") + "/";
|
||||||
@ -255,7 +256,7 @@ app.use(basePath, function(req, res, next) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
/* Everything below here requires a successful authentication */
|
/* Everything below here requires a successful authentication */
|
||||||
app.use(basePath, express.static(picturesPath, { index: false }));
|
app.use(basePath, express.static(frontendPath, { index: false }));
|
||||||
|
|
||||||
app.use(basePath + "api/v1/games", require("./routes/games"));
|
app.use(basePath + "api/v1/games", require("./routes/games"));
|
||||||
|
|
||||||
@ -7,8 +7,8 @@ module.exports = merge(common, {
|
|||||||
mode: "development",
|
mode: "development",
|
||||||
devServer: {
|
devServer: {
|
||||||
contentBase: path.join(__dirname, "/"),
|
contentBase: path.join(__dirname, "/"),
|
||||||
port: 8008,
|
port: 8930,
|
||||||
publicPath: "http://localhost:8008/dist/",
|
publicPath: "http://localhost:8930/dist/",
|
||||||
hotOnly: true,
|
hotOnly: true,
|
||||||
disableHostCheck: true,
|
disableHostCheck: true,
|
||||||
historyApiFallback: true
|
historyApiFallback: true
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user