Fixed unhandled promises rejection issues

.gitignore

@@ -1,3 +1,6 @@
 node_modules
 output
+cache
 .env
+.DS_Store
+*.code-workspace

README.md

@@ -1,18 +1,20 @@
 This is a tool to scrape World of Warcraft realm and realm-connection data from the Blizzard Developer API, primarily for use in the [LibRealmInfo](https://github.com/phanx-wow/LibRealmInfo) library.
 
-### Usage
+## Usage
 
 1. Run `yarn start` or `npm start` or `node index.js`.
 2. See the generated `connectionData.json` and `realmData.json` files in the `output` directory.
 
-Total running time is over 8 minutes, as a delay of 12 ms is enforced between API calls in order to comply with the "100 requests per second" limit for free accounts.
+Total running time is over 8 minutes, as a delay of 250 ms is enforced between API calls in order to comply with the "100 requests per second" limit for free accounts.
 
-#### For use in LibRealmInfo:
+In case of failure (ie due to a timeout error), the script will stop. However, the already fetched data is stored in the `cache` folder so you should be able to relaunch it and resume the process.
 
-1. Optionally, run `yarn convert` or `npm convert` or `node convert.js`.
+### For use in LibRealmInfo
+
+1. Run `yarn convert` or `npm convert` or `node convert.js`.
 2. See the generated `data.lua` file in the `output` directory.
 
-### Requirements
+## Requirements
 
 1. [Install Node.js](https://nodejs.org/)
 2. Optionally, [install Yarn](https://yarnpkg.com/lang/en/docs/install/)
@@ -24,6 +26,10 @@ Total running time is over 8 minutes, as a delay of 12 ms is enforced between AP
 	CLIENT_SECRET=<your Blizzard API secret>
 	```
 
-### License
+## Known issues
+
+The realms API doesn't seem to be implemented for China since it only returns HTTP 403 Forbidden errors. Chinese server information was set manually under the `data` folder and probably needs to be updated.
+
+## License
 
 Zlib license. See the `LICENSE` file for the full text.

connections.js

@@ -1,25 +1,41 @@
 const fetch = require("node-fetch")
-const sleep = require("sleep").msleep
+const jsonfile = require("jsonfile")
+const fs = require("fs")
 
-const { SLEEP_TIME, apiDomains } = require("./constants.js")
+const { SLEEP_TIME, apiDomains, namespaces } = require("./constants.js")
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
 
 const getConnection = async (requestOptions, region, id) => {
-	sleep(SLEEP_TIME)
+	const filename = `cache/connection-${region}-${id}.json`
+
+	// Get JSON file from cache
+	if (fs.existsSync(filename)) {
+		return jsonfile.readFileSync(filename)
+	}
+
+	// Fetch new connection data
+	await sleep(SLEEP_TIME)
 
 	const url = "https://" + apiDomains[region] + "/data/wow/connected-realm/" + id
 	const res = await fetch(url, requestOptions)
 	const json = await res.json()
-	if (!json.realms) return console.log("ERROR: response missing realms:", json)
 
-	return {
+	if (!json.realms && json.code !== 404) return console.log("ERROR: response missing realms:", json)
+
+	const connectionData = {
 		id    : id,
 		region: region.toUpperCase(),
-		realms: json.realms.map(realm => realm.id),
+		realms: json.realms ? json.realms.map(realm => realm.id) : [],
 	}
+
+	jsonfile.writeFileSync(filename, connectionData, { spaces: "\t" }, (err) => console.error(err))
+
+	return connectionData
 }
 
 const getConnectionIDsForRegion = async (requestOptions, region) => {
-	sleep(SLEEP_TIME)
+	await sleep(SLEEP_TIME)
 
 	const url = "https://" + apiDomains[region] + "/data/wow/connected-realm/"
 	const res = await fetch(url, requestOptions)
@@ -32,12 +48,12 @@ const getConnectionIDsForRegion = async (requestOptions, region) => {
 	})
 }
 
-const getConnectionsForRegion = async (accessToken, region) => {
+const getConnectionsForRegion = async (accessToken, region, namespace) => {
 	const list = []
 	const requestOptions = {
 		headers: {
 			"Authorization": "Bearer " + accessToken,
-			"Battlenet-Namespace": "dynamic-" + region,
+			"Battlenet-Namespace": namespace + "-" + region,
 		}
 	}
 

constants.js

@@ -1,7 +1,7 @@
-exports.SLEEP_TIME = 12 // delay to enforce between API calls
+exports.SLEEP_TIME = 15 // delay to enforce between API calls
 
 exports.regions = [
-	// "cn", // there is now an API but it doesn't allow me access
+	// "cn", // CN API is not implemented. Will be using static data instead
 	"eu",
 	"kr",
 	"tw",
@@ -32,14 +32,27 @@ exports.authRegions = {
 	us: "us",
 },
 
+exports.namespaces = {
+	'dynamic': '',
+	'dynamic-classic1x': 'classic1x',
+	'dynamic-classic': 'classic',
+},
+
+exports.ruleCodes = {
+	'Roleplaying': 'RP',
+	'Normal': 'PvE',
+	'PvP': 'PvP',
+	'PvP RP': 'PvP RP',
+},
+
 exports.timezones = {
 	"America/Chicago": "CST", // US Central
 	"America/Denver": "MST", // US Mountain
 	"America/Los_Angeles": "PST", // US Pacific
 	"America/New_York": "EST", // US Eastern
-	"America/Sao_Paolo": "BRT", // Brazil
+	"America/Sao_Paulo": "BRT", // Brazil
 	"Asia/Seoul": "", // Korea
-	"Asia/Taipei": "", // Hong Kong
+	"Asia/Taipei": "", // Taiwan
 	"Australia/Melbourne": "AEST", // Oceanic
 	"Europe/Paris": "", // EU Central
 }

convert.js

@@ -1,6 +1,7 @@
 const fs = require("fs")
 const realmData = require("./output/realmData.json")
 const connectionData = require("./output/connectionData.json")
+const { timezones, ruleCodes } = require("./constants.js")
 
 const sortById = (a, b) => a.id < b.id ? -1 : 1
 
@@ -12,8 +13,10 @@ console.log("Converting realm data...")
 
 realmData.sort(sortById).forEach(realm => {
 	let { id, englishName, locale, name, region, rules, timezone } = realm
+	rules = ruleCodes[rules] || rules
 	if (region === "US") {
 		// [id] = "name,rules,locale,region,timezone"
+		timezone = timezones[timezone] || timezone
 		output.push(`[${id}]="${name},${rules},${locale},${region},${timezone}",`)
 	} else if (englishName && englishName !== name) {
 		// [id] = "name,rules,locale,region,englishName"
@@ -33,8 +36,12 @@ output.push("connectionData = {")
 console.log("Converting connection data...")
 
 connectionData.sort(sortById).forEach(connection => {
-	const { id, region, realms } = connection
-	output.push(`"${id},${region},${realms.join(",")}",`)
+	let { id, region, realms } = connection
+	realms.sort((a, b) => a - b)
+	realms = [...realms.filter(realmId => realmId === id), ...realms.filter(realmId => realmId !== id)]
+	if (realms.length > 0) {
+		output.push(`"${id},${region},${realms.join(",")}",`)
+	}
 })
 
 output.push("}")