500 lines
16 KiB
JavaScript
500 lines
16 KiB
JavaScript
const fs = require("fs").promises,
|
|
fetch = require("node-fetch"),
|
|
FormData = require("form-data"),
|
|
{ JSDOM } = require("jsdom"),
|
|
HttpsProxyAgent = require('http-proxy-agent'),
|
|
{ request, GraphQLClient, gql } = require("graphql-request"),
|
|
moment = require("moment");
|
|
|
|
/* Any search URL to oregonliquorsearch that has an expired cookie
|
|
* will take you back to the main "verify you are 21" page
|
|
*
|
|
* If you hit the server too frequently, or perhaps too many times
|
|
* with the same cookie, it will ban your IP for an indeterminate
|
|
* amount of time. It might be a scaled value; I've been blocked
|
|
* for an hour before or several days other times.
|
|
*/
|
|
|
|
const cookies = [];
|
|
|
|
const client = new GraphQLClient("http://localhost:4000/graphql", { headers: {} });
|
|
|
|
const getWhiskyInfo = async (whisky) => {
|
|
const cacheFile = `cache/${whisky.code}`;
|
|
return fs.stat(cacheFile).then(async stat => {
|
|
const last = moment(stat.mtime),
|
|
yesterday = moment().subtract(1, 'day');;
|
|
|
|
if (last.isBefore(yesterday)) {
|
|
console.log(`${cacheFile} ${last.calendar()} is older than yesterday at this time: ${yesterday.calendar()}`);
|
|
await fs.unlink(cacheFile);
|
|
} else {
|
|
console.log(`${cacheFile} ${last.calendar()} is newer than yesterday at this time: ${yesterday.calendar()}`);
|
|
}
|
|
|
|
return fs.readFile(cacheFile);
|
|
})
|
|
.catch(async (error) => {
|
|
if (criticalFailure) throw error;
|
|
console.log(`Loading new location data for ${whisky.code}`);
|
|
|
|
const url = `http://oregonliquorsearch.com/servlet/FrontController` +
|
|
`?radiusSearchParam=10` +
|
|
`&productSearchParam=${whisky.code}` +
|
|
`&locationSearchParam=97225` +
|
|
`&btnSearch=Search` +
|
|
`&view=global` +
|
|
`&action=search`;
|
|
|
|
await rateLimit();
|
|
return fetch(url, {
|
|
headers: {
|
|
cookie: cookies
|
|
}
|
|
})
|
|
.then(response => response.text())
|
|
.then(dom => {
|
|
try {
|
|
fs.writeFile(cacheFile, dom);
|
|
} catch (_error) {
|
|
if (criticalFailure) throw _error;
|
|
console.log(`Write failed: ${_error}`);
|
|
}
|
|
return dom;
|
|
});
|
|
}).then(dom => {
|
|
dom = new JSDOM(dom);
|
|
const headers = [];
|
|
|
|
const rows = dom.window.document.querySelectorAll('table.list tr');
|
|
const olccLocations = [];
|
|
rows.forEach(row => {
|
|
const th = row.querySelectorAll('table.list th');
|
|
if (th.length) {
|
|
th.forEach(header => headers.push(header.textContent.trim()));
|
|
return;
|
|
}
|
|
|
|
const tds = row.querySelectorAll('td'), location = {};
|
|
tds.forEach((td, index) => {
|
|
const span = td.querySelector('span.link');
|
|
let value = (span ? span.textContent : td.textContent).trim();
|
|
if (parseFloat(value) == value) {
|
|
value = parseFloat(value);
|
|
}
|
|
location[headers[index]] = value;
|
|
});
|
|
olccLocations.push(location);
|
|
});
|
|
return olccLocations;
|
|
})
|
|
.then(async (olccLocations) => {
|
|
console.log(`Updating locations for ${whisky.code}`);
|
|
const whiskyInventories = []; /* Locations OLCC reports for this whisky */
|
|
await Promise.all(olccLocations.map(async (olccLocation) => {
|
|
let location = locations.find(item => item.code == olccLocation['Store No']);
|
|
let operation;
|
|
|
|
if (location == null) { /* New location */
|
|
console.log(`NEW location: ${olccLocation['Store No']} ${olccLocation['Address']} ${olccLocation['Location']}`);
|
|
location = {
|
|
code: olccLocation['Store No'].toString(),
|
|
address: olccLocation['Address'],
|
|
city: olccLocation['Location'],
|
|
phone: olccLocation['Telephone'],
|
|
longitude: -110.5,
|
|
latitude: 45
|
|
};
|
|
operation = gql`
|
|
mutation CreateLocation($code: String!, $address: String!, $city: String!, $phone: String!, $longitude: Float!, $latitude: Float!) {
|
|
createLocation(code: $code, address: $address, city: $city, phone: $phone, longitude: $longitude, latitude: $latitude) {
|
|
code
|
|
}
|
|
}`;
|
|
} else {
|
|
operation = gql`
|
|
mutation UpdateLocation($code: String!, $address: String!, $city: String!, $phone: String!, $longitude: Float!, $latitude: Float!) {
|
|
updateLocation(code: $code, address: $address, city: $city, phone: $phone, longitude: $longitude, latitude: $latitude)
|
|
}`;
|
|
}
|
|
|
|
try {
|
|
const data = await client.request(operation, location);
|
|
if (data.createLocation && data.createLocation.code) {
|
|
locations.push(location);
|
|
}
|
|
whiskyInventories.push({
|
|
location,
|
|
updated: moment().format("YYYY-MM-DD"),
|
|
quantity: olccLocation['Qty']
|
|
});
|
|
} catch (error) {
|
|
console.error(JSON.stringify(error, null, 2));
|
|
console.log(location);
|
|
process.exit(1);
|
|
}
|
|
}));
|
|
|
|
/* Manage Inventories table */
|
|
/* Delete entries on whisky which do not exist in the new OLCC locations for this whisky, or update quantity for those that do */
|
|
await Promise.all(whisky.inventories.map(async (inventory) => {
|
|
const update = whiskyInventories.find(item => item.location.code == inventory.location.code);
|
|
let operation, variables;
|
|
if (update) {
|
|
Object.assign(inventory, update);
|
|
operation = gql`
|
|
mutation UpdateInventory($location: String!, $whisky: String!, $quantity: Int!, $updated: String!) {
|
|
updateInventory(location: $location, whisky: $whisky, quantity: $quantity, updated: $updated)
|
|
}`;
|
|
//console.log("update", JSON.stringify(inventory));
|
|
variables = {
|
|
whisky: whisky.code,
|
|
location: update.location.code,
|
|
quantity: update.quantity,
|
|
updated: moment().format("YYYY-MM-DD")
|
|
};
|
|
} else {
|
|
operation = gql`
|
|
mutation DeleteInventory($location: String!, $whisky: String!) {
|
|
deleteInventory(location: $location, whisky: $whisky)
|
|
}`;
|
|
console.log("delete", JSON.stringify(inventory));
|
|
variables = {
|
|
whisky: whisky.code,
|
|
location: inventory.location.code,
|
|
};
|
|
}
|
|
try {
|
|
await client.request(operation, variables);
|
|
} catch (error) {
|
|
console.error(JSON.stringify(error, null, 2));
|
|
console.log(JSON.stringify(variables, null, 2), update);
|
|
process.exit(1);
|
|
}
|
|
}));
|
|
/* If an entry exists in the OLCC locations for this whisky, and not on the whisky, then add a new Inventory entry */
|
|
await Promise.all(whiskyInventories.map(async (olccInventory) => {
|
|
const update = whisky.inventories.find(item => item.location.code == olccInventory.location.code);
|
|
if (!update) {
|
|
const operation = gql`
|
|
mutation CreateInventory($location: String!, $whisky: String!, $quantity: Int!, $updated: String!) {
|
|
createInventory(location: $location, whisky: $whisky, quantity: $quantity, updated: $updated) {
|
|
quantity
|
|
}
|
|
}`;
|
|
try {
|
|
console.log(JSON.stringify(olccInventory, null, 2));
|
|
const data = await client.request(operation, {
|
|
whisky: whisky.code,
|
|
location: olccInventory.location.code,
|
|
quantity: olccInventory.quantity,
|
|
updated: moment().format("YYYY-MM-DD")
|
|
});
|
|
//console.log(`New inventory: ${JSON.stringify(data, null, 2)}`);
|
|
whisky.inventories.push(data.createInventory.inventory);
|
|
} catch (error) {
|
|
console.log('there');
|
|
console.error(JSON.stringify(error, null, 2));
|
|
console.log(operation);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
}));
|
|
whisky.inventories = whiskyInventories;
|
|
return whisky.inventories;
|
|
});
|
|
};
|
|
|
|
console.log(`Trying from "cache"...`);
|
|
|
|
let lastFetch = undefined;
|
|
const rateLimit = async () => {
|
|
if (!lastFetch) {
|
|
lastFetch = Date.now();
|
|
return;
|
|
}
|
|
const delay = (lastFetch + (Math.random() * 5000 + 2500)) - Date.now();
|
|
if (delay > 0) {
|
|
await wait(delay);
|
|
}
|
|
lastFetch = Date.now();
|
|
}
|
|
|
|
const wait = async (timeout) => {
|
|
return new Promise((resolve, reject) => {
|
|
setTimeout(resolve, timeout);
|
|
});
|
|
};
|
|
|
|
let criticalFailure = false;
|
|
|
|
let whiskies = [],
|
|
locations = [];
|
|
|
|
fs.stat("cache")
|
|
.catch((error) => {
|
|
if (criticalFailure) throw error;
|
|
fs.mkdir("cache");
|
|
return fs.stat("cache");
|
|
})
|
|
.then(async (stats) => {
|
|
if (!stats.isDirectory()) {
|
|
criticalFailure = true;
|
|
throw new Error("cache is not a directory or can not be created");
|
|
}
|
|
|
|
let query = gql` {
|
|
Whiskies {
|
|
code
|
|
description
|
|
price
|
|
size
|
|
lastSeen
|
|
inventories {
|
|
location {
|
|
code
|
|
address
|
|
city
|
|
longitude
|
|
latitude
|
|
}
|
|
quantity
|
|
updated
|
|
}
|
|
}
|
|
}`;
|
|
|
|
try {
|
|
whiskies = (await client.request(query)).Whiskies;
|
|
} catch (error) {
|
|
console.error(error);
|
|
process.exit(-1);
|
|
}
|
|
|
|
query = gql` {
|
|
Locations {
|
|
code
|
|
address
|
|
city
|
|
phone
|
|
longitude
|
|
latitude
|
|
}
|
|
}`;
|
|
|
|
try {
|
|
locations = (await client.request(query)).Locations;
|
|
} catch (error) {
|
|
console.error(error);
|
|
process.exit(-1);
|
|
}
|
|
})
|
|
.catch(error => console.error(error))
|
|
.then(async () => {
|
|
/* Attempt to read the whisky cache data.
|
|
* If it is older than 1 day, fetch new data */
|
|
const cacheFile = "cache/whiskies",
|
|
stat = await fs.stat(cacheFile),
|
|
last = moment(stat.mtime),
|
|
yesterday = moment().subtract(1, 'day');
|
|
|
|
if (last.isBefore(yesterday)) {
|
|
console.log(`${cacheFile} ${last.calendar()} is older than yesterday at this time: ${yesterday.calendar()}`);
|
|
await fs.unlink(cacheFile);
|
|
} else {
|
|
console.log(`${cacheFile} ${last.calendar()} is newer than yesterday at this time: ${yesterday.calendar()}`);
|
|
}
|
|
|
|
const data = await fs.readFile(cacheFile);
|
|
return fs.readFile("cache/cookies")
|
|
.then(cookieData => {
|
|
JSON.parse(cookieData).forEach(cookie => cookies.push(cookie))
|
|
})
|
|
.then(() => {
|
|
return data;
|
|
});
|
|
}).catch(async (error) => {
|
|
if (criticalFailure) throw error;
|
|
console.error(error);
|
|
console.log(`Cache or cookies not found--loading fresh whisky list and cookies.`);
|
|
await rateLimit();
|
|
return fetch("http://oregonliquorsearch.com/", {
|
|
// agent: new HttpsProxyAgent('http://3.22.0.212:8080')
|
|
}).then((response) => {
|
|
response.headers.raw()['set-cookie']
|
|
.forEach((header) => {
|
|
header
|
|
.split(';')
|
|
.forEach(cookie => {
|
|
cookies.push(cookie);
|
|
});
|
|
});
|
|
fs.writeFile("cache/cookies", JSON.stringify(cookies));
|
|
})
|
|
.then(async () => {
|
|
/* if (!cookies.JSESSIONID) {
|
|
throw "JSESSIONID not found";
|
|
}*/
|
|
console.log(`Submitting age verification...`);
|
|
const formData = new FormData();
|
|
formData.append("btnSubmit", "I'm 21 or older");
|
|
await rateLimit();
|
|
return fetch("http://oregonliquorsearch.com/servlet/WelcomeController", {
|
|
method: "POST",
|
|
body: formData,
|
|
headers: {
|
|
cookie: cookies
|
|
}
|
|
})
|
|
.then((response) => {
|
|
if (response.status != 200) {
|
|
throw new Error("Error fetching resource from oregonliquorsearch");
|
|
}
|
|
});
|
|
}).then(async () => {
|
|
/* Search for all WHISKY types */
|
|
console.log(`Searching for all whisky types...`);
|
|
await rateLimit();
|
|
return fetch("http://oregonliquorsearch.com/servlet/FrontController" +
|
|
"?view=browsecategoriesallsubcategories" +
|
|
"&action=select" +
|
|
"&category=DOMESTIC%20WHISKEY", {
|
|
headers: {
|
|
cookie: cookies
|
|
}
|
|
})
|
|
.then(response => response.text());
|
|
})
|
|
.then(async (dom) => {
|
|
try {
|
|
console.log(`Writing to "cache"...`);
|
|
await fs.writeFile("cache/whiskies", dom);
|
|
} catch(_error) {
|
|
if (criticalFailure) throw _error;
|
|
console.log(`Write failed: ${_error}`);
|
|
}
|
|
return dom;
|
|
});
|
|
}).then(dom => {
|
|
dom = new JSDOM(dom);
|
|
const headers = [];
|
|
|
|
const rows = dom.window.document.querySelectorAll('table.list tr');
|
|
const olccWhiskies = [];
|
|
const processRow = (row) => {
|
|
const th = row.querySelectorAll('table.list th a');
|
|
if (th.length) {
|
|
th.forEach(header => headers.push(header.textContent.trim()));
|
|
return;
|
|
}
|
|
const tds = row.querySelectorAll('td'), whisky = {};
|
|
tds.forEach((td, index) => {
|
|
let value = td.textContent.trim();
|
|
|
|
switch (headers[index]) {
|
|
case 'Size':
|
|
if (value.match(/^.* ML/)) {
|
|
value = parseFloat(value) / 1000.;
|
|
} else if (value.match(/^.* L/)) {
|
|
value = parseFloat(value);
|
|
} else if (value == 'LITER') {
|
|
value = 1.;
|
|
}
|
|
break;
|
|
|
|
case 'Proof':
|
|
if (parseFloat(value) == value) {
|
|
value = parseFloat(value);
|
|
}
|
|
break;
|
|
|
|
case 'Case Price':
|
|
case 'Bottle Price':
|
|
if (value.match(/^\$/)) {
|
|
value = parseFloat(value.replace(/^\$/, ""));
|
|
}
|
|
break;
|
|
|
|
case 'Item Code':
|
|
const anchor = td.querySelector('a');
|
|
if (anchor) {
|
|
whisky.url = `http://oregonliquorsearch.com${anchor.href}`;
|
|
}
|
|
value = td.querySelector('span.link').textContent.trim();
|
|
break;
|
|
|
|
case 'Description':
|
|
case 'Age':
|
|
default:
|
|
break;
|
|
}
|
|
whisky[headers[index]] = value;
|
|
});
|
|
olccWhiskies.push(whisky);
|
|
};
|
|
|
|
rows.forEach(processRow);
|
|
return olccWhiskies;
|
|
}).then(async (olccWhiskies) => {
|
|
await Promise.all(olccWhiskies.map(async (olccWhisky) => {
|
|
let whisky = whiskies.find(item => item.code == olccWhisky['Item Code']);
|
|
let operation;
|
|
|
|
if (whisky == null) { /* New whisky */
|
|
console.log(`NEW whisky: ${olccWhisky['Item Code']} ${olccWhisky['Description']}`);
|
|
whisky = {
|
|
description: olccWhisky['Description'],
|
|
code: olccWhisky['Item Code'].toString(),
|
|
size: Math.floor(1000 * olccWhisky['Size']),
|
|
proof: olccWhisky['Proof'],
|
|
age: olccWhisky['Age'],
|
|
price: olccWhisky['Bottle Price'],
|
|
};
|
|
operation = gql`
|
|
mutation CreateWhisky($description: String!, $code: String!, $size: Int!, $price: Float!, $lastSeen: String!) {
|
|
createWhisky(description: $description, code: $code, size: $size, price: $price, lastSeen: $lastSeen) {
|
|
code
|
|
}
|
|
}`;
|
|
} else {
|
|
operation = gql`
|
|
mutation UpdateWhisky($description: String!, $code: String!, $size: Int!, $price: Float!, $lastSeen: String!) {
|
|
updateWhisky(description: $description, code: $code, size: $size, price: $price, lastSeen: $lastSeen)
|
|
}`;
|
|
}
|
|
|
|
whisky.lastSeen = moment().format("YYYY-MM-DD");
|
|
|
|
try {
|
|
const data = await client.request(operation, whisky);
|
|
if (whisky.code === undefined) {
|
|
whisky.code = data;
|
|
}
|
|
} catch (error) {
|
|
console.error(JSON.stringify(error, null, 2));
|
|
console.error(operation);
|
|
console.error(variables);
|
|
process.exit(1);
|
|
}
|
|
}));
|
|
|
|
console.log(`OLCC currently carries ${whiskies.length} domestic whiskies.`);
|
|
whiskies = whiskies
|
|
.filter(a => a.code == '2170B')
|
|
.filter(a => a.price >= 10.)// && a['Bottle Price'] <= 100.)
|
|
.sort((a, b) => b.price - a.price);
|
|
const max = Math.min(5, whiskies.length);
|
|
console.log(`${max} whiskies > $10:`);
|
|
for (let i = 0; i < max; i++) {
|
|
console.log(`${i+1}. \$${whiskies[i].price} - ${whiskies[i].description}`);
|
|
const _inventories = await getWhiskyInfo(whiskies[i]);
|
|
_inventories.sort((a, b) => b.quantity - a.quantity);
|
|
/* Create or Update location */
|
|
/* Update location inventory */
|
|
_inventories.forEach((inventory) => {
|
|
console.log(` ${inventory.quantity} at ${inventory.location.address} in ${inventory.location.city}, OR`);
|
|
});
|
|
}
|
|
});
|
|
|