diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..47488040 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "eslint.enable": false +} \ No newline at end of file diff --git a/README.md b/README.md index fbd6615b..3d3913b8 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ user.storage.on('read', function(filename, callback) { }); ``` -In this manner, you can save data to a database, a cloud service, or anything else you choose. +In this manner, you can save data to a database, a cloud service, or anything else you choose. If both [`savePicsCache`](#savePicsCache) and [`picsCacheAll`](#picsCacheAll) are enabled, it is possible to maintain your own Steam database this way. ### autoRelogin @@ -175,6 +175,16 @@ Added in 3.3.0. Defaults to `false`. +### savePicsCache + +If `enablePicsCache` is enabled, saves all product info from [the PICS cache](#picscache) to disk (in [`dataDirectory`](#dataDirectory)) or to your [Custom Storage Engine](#custom-storage-engine). This will significantly speed up the [`appOwnershipCached`](#appOwnershipCache) event from firing and reduce the amount product info requests to Steam. It will only save product info if it's not missing its access token. + +Added in {{TODO}} + +Defaults to `false`. + +**Warning:** Mind that this will significantly increase the storage usage and space! + ### picsCacheAll If `picsCacheAll` is enabled, `enablePicsCache` is enabled, and `changelistUpdateInterval` is nonzero, then apps and @@ -879,6 +889,7 @@ Requests a list of game servers from the master server. - `callback` - Called when requested data is available - `err` - An `Error` object on failure, or `null` on success - `apps` - An object whose keys are AppIDs and whose values are objects + - `sha` - The SHA hash of the app info in hex format (useful to compare to old data) - `changenumber` - The changenumber of the latest changelist in which this app changed - `missingToken` - `true` if you need to provide an access token to get more details about this app - `appinfo` - An object whose structure is identical to the output of `app_info_print` in the [Steam console](steam://nav/console) diff --git a/components/05-filestorage.js b/components/05-filestorage.js index 8f271ccf..c4fe8e25 100644 --- a/components/05-filestorage.js +++ b/components/05-filestorage.js @@ -17,6 +17,19 @@ class SteamUserFileStorage extends SteamUserUtility { } } + /** + * @param {Object} files - Keys are filenames, values are Buffer objects containing the file contents + * @return {Promise} + * @protected + */ + async _saveFiles(files) { + if (!this.storage) { + return Promise.reject(new Error('Storage system disabled')); + } + + return await this.storage.saveFiles(files); + }; + /** * @param {string} filename * @returns {Promise} diff --git a/components/apps.js b/components/apps.js index 206b50eb..82f7c0d1 100644 --- a/components/apps.js +++ b/components/apps.js @@ -60,19 +60,19 @@ class SteamUserApps extends SteamUserAppAuth { let processedApps = apps.map((app) => { if (typeof app == 'string') { - app = {game_id: '15190414816125648896', game_extra_info: app}; + app = { game_id: '15190414816125648896', game_extra_info: app }; } else if (typeof app != 'object') { - app = {game_id: app}; + app = { game_id: app }; } if (typeof app.game_ip_address == 'number') { - app.game_ip_address = {v4: app.game_ip_address}; + app.game_ip_address = { v4: app.game_ip_address }; } return app; }); - this._send(EMsg.ClientGamesPlayedWithDataBlob, {games_played: processedApps}); + this._send(EMsg.ClientGamesPlayedWithDataBlob, { games_played: processedApps }); processedApps.forEach((app) => { if (app.game_id > Math.pow(2, 32)) { @@ -113,7 +113,7 @@ class SteamUserApps extends SteamUserAppAuth { } else if (blocked) { return reject(new Error('Cannot kick other session')); } else { - return resolve({playingApp}); + return resolve({ playingApp }); } }); }); @@ -127,12 +127,12 @@ class SteamUserApps extends SteamUserAppAuth { */ getPlayerCount(appid, callback) { return StdLib.Promises.timeoutCallbackPromise(10000, ['playerCount'], callback, (resolve, reject) => { - this._send(EMsg.ClientGetNumberOfCurrentPlayersDP, {appid}, (body) => { + this._send(EMsg.ClientGetNumberOfCurrentPlayersDP, { appid }, (body) => { let err = Helpers.eresultError(body.eresult); if (err) { reject(err); } else { - resolve({playerCount: body.player_count}); + resolve({ playerCount: body.player_count }); } }); }); @@ -142,7 +142,7 @@ class SteamUserApps extends SteamUserAppAuth { * Get a list of apps or packages which have changed since a particular changenumber. * @param {int} sinceChangenumber - Changenumber to get changes since. Use 0 to get the latest changenumber, but nothing else * @param {function} [callback] - * @returns {Promise<{currentChangeNumber: number, appChanges: number[], packageChanges: number[]}>} + * @returns {Promise<{currentChangeNumber: number, appChanges: object[], packageChanges: object[]}>} */ getProductChanges(sinceChangenumber, callback) { let args = ['currentChangeNumber', 'appChanges', 'packageChanges']; @@ -161,28 +161,148 @@ class SteamUserApps extends SteamUserAppAuth { }); } + /** + * @param {apps: Object, packages: Object} - Response from getProductInfo + * @returns {Promise} + * @protected + */ + _saveProductInfo({ apps, packages }) { + if (!this.options.savePicsCache) { + return Promise.resolve([]); + } + + let toSave = {}; + + for (let appid in apps) { + // We want to avoid saving apps that are missing a token + // Public only apps are weird... + if (apps[appid].missingToken && !apps[appid].appinfo.public_only) { + continue; + } + let filename = `app_info_${appid}.json`; + let contents = JSON.stringify(apps[appid]); + toSave[filename] = contents; + } + + for (let packageid in packages) { + if (packages[packageid].missingToken) { + continue; + } + let filename = `package_info_${packageid}.json`; + let contents = JSON.stringify(packages[packageid]); + toSave[filename] = contents; + } + + return this._saveFiles(toSave); + } + + /** + * Get cached product info. + * @param {object[]} apps - Array of apps {appid, access_token} + * @param {object[]} packages - Array of packages {packageid, access_token} + * @returns {Promise<{apps: Object, packages: Object, notCachedApps: number[], notCachedPackages: number[]}>} + * @protected + */ + async _getCachedProductInfo(apps, packages) { + let response = { + apps: {}, + packages: {}, + notCachedApps: [], + notCachedPackages: [] + }; + + // With pics cache disabled, we cannot assure pics cache is up to date. + if (!this.options.enablePicsCache) { + return response; + } + + // From this point, we can assume pics cache is up to date (via changelist updates). + for (let { appid } of apps) { + if (this.picsCache.apps[appid]) { + response.apps[appid] = this.picsCache.apps[appid]; + } else { + response.notCachedApps.push(appid); + } + } + for (let { packageid } of packages) { + if (this.picsCache.packages[packageid]) { + response.packages[packageid] = this.picsCache.packages[packageid]; + } else { + response.notCachedPackages.push(packageid); + } + } + + // If everything was already in memory cache, we're done. + if (response.notCachedApps.length === 0 && response.notCachedPackages.length === 0) { + return response; + } + + // If pics cache is not being saved to disk, we're done. + if (!this.options.savePicsCache) { + return response; + } + + // Otherwise, we try loading the missing apps & packages from disk. + let appids = response.notCachedApps; + let packageids = response.notCachedPackages; + response.notCachedApps = []; + response.notCachedPackages = []; + + let appFiles = {}; + let packageFiles = {}; + for (let appid of appids) { + let filename = `app_info_${appid}.json`; + appFiles[filename] = appid; + } + for (let packageid of packageids) { + let filename = `package_info_${packageid}.json`; + packageFiles[filename] = packageid; + } + let files = await this._readFiles(Object.keys(appFiles).concat(Object.keys(packageFiles))); + + for (let { filename, error, contents } of files) { + if (Buffer.isBuffer(contents)) { + contents = contents.toString('utf8'); + } + let appid = appFiles[filename]; + let packageid = packageFiles[filename]; + + if (appid !== undefined) { + if (error || !contents) { + response.notCachedApps.push(appid); + } else { + // Save to memory cache + this.picsCache.apps[appid] = JSON.parse(contents); + response.apps[appid] = this.picsCache.apps[appid]; + } + } else if (packageid !== undefined) { // Remember, package ID can be 0 + if (error || !contents) { + response.notCachedPackages.push(packageid); + } else { + this.picsCache.packages[packageid] = JSON.parse(contents); + response.packages[packageid] = this.picsCache.packages[packageid]; + } + } else { + this.emit('debug', `Error retrieving origins of file ${filename}`); + } + } + + return response; + } + /** * Get info about some apps and/or packages from Steam. * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} * @param {int[]|object[]} packages - Array of package IDs. May be empty. May also contain objects with keys {packageid, access_token} - * @param {boolean} [inclTokens=false] - If true, automatically retrieve access tokens if needed - * @param {function} [callback] + * @param {boolean} inclTokens - If true, automatically retrieve access tokens if needed + * @param {function} callback * @param {int} [requestType] - Don't touch * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} + * @protected */ - getProductInfo(apps, packages, inclTokens, callback, requestType) { - // Adds support for the previous syntax - if (typeof inclTokens !== 'boolean' && typeof inclTokens === 'function') { - requestType = callback; - callback = inclTokens; - inclTokens = false; - } - - // This one actually can take a while, so allow it to go as long as 60 minutes - return StdLib.Promises.timeoutCallbackPromise(3600000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, (resolve, reject) => { - requestType = requestType || PICSRequestType.User; - - // Steam can send us the full response in multiple responses, so we need to buffer them into one callback + _getProductInfo(apps, packages, inclTokens, callback, requestType = PICSRequestType.User) { + // Steam can send us the full response in multiple responses, so we need to buffer them into one callback + return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { let appids = []; let packageids = []; let response = { @@ -191,39 +311,70 @@ class SteamUserApps extends SteamUserAppAuth { unknownApps: [], unknownPackages: [] }; + let cached = { + apps: {}, + packages: {}, + notCachedApps: [], + notCachedPackages: [] + }; + let shaList = { + apps: {}, + packages: {} + }; - apps = apps.map((app) => { + // Preprocess input: apps and packages + let _apps = []; + for (let app of apps) { + let appid = parseInt(typeof app === 'object' ? app.appid : app, 10); + // Ensure uniqueness to prevent nasty bugs + if (appids.includes(appid)) { + continue; + } if (typeof app === 'object') { + app.appid = appid; appids.push(app.appid); - return app; } else { - appids.push(app); - return {appid: app}; + appids.push(appid); + app = { appid }; + } + _apps.push(app); + } + apps = _apps; + + let _packages = []; + for (let pkg of packages) { + let packageid = parseInt(typeof pkg === 'object' ? pkg.packageid : pkg, 10); + // Ensure uniqueness to prevent nasty bugs + if (packageids.includes(packageid)) { + continue; } - }); - - packages = packages.map((pkg) => { if (typeof pkg === 'object') { + pkg.packageid = packageid; packageids.push(pkg.packageid); - return pkg; } else { - packageids.push(pkg); - return {packageid: pkg}; + packageids.push(packageid); + pkg = { packageid }; } - }); - - if (inclTokens) { - packages.filter(pkg => !pkg.access_token).forEach((pkg) => { + if (inclTokens && !pkg.access_token) { // Check if we have a license for this package which includes a token let license = (this.licenses || []).find(lic => lic.package_id == pkg.packageid && lic.access_token != 0); if (license) { this.emit('debug', `Using token "${license.access_token}" from license for package ${pkg.packageid}`); pkg.access_token = license.access_token; } - }); + } + _packages.push(pkg); } + packages = _packages; - this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, async (body) => { + // If we have no apps or packages, we're done + if (appids.length === 0 && packageids.length === 0) { + resolve(response); + return; + } + + // Function to handle response of ClientPICSProductInfoRequest (may be called multiple times) + let onResponse = async (body) => { // If we're using the PICS cache, then add the items in this response to it if (this.options.enablePicsCache) { let cache = this.picsCache; @@ -231,15 +382,20 @@ class SteamUserApps extends SteamUserAppAuth { cache.packages = cache.packages || {}; (body.apps || []).forEach((app) => { - let appInfoVdf = app.buffer.toString('utf8'); - // It seems that Steam appends a NUL byte. Unsure if this is universal or not, but to make sure - // that things work regardless of whether there's a NUL byte at the end, just remove it if it's there. - appInfoVdf = appInfoVdf.replace(/\0$/, ''); + let appinfo = null; + if (app.buffer) { + let appInfoVdf = app.buffer.toString('utf8'); + // It seems that Steam appends a NUL byte. Unsure if this is universal or not, but to make sure + // that things work regardless of whether there's a NUL byte at the end, just remove it if it's there. + appInfoVdf = appInfoVdf.replace(/\0$/, ''); + appinfo = VDF.parse(appInfoVdf).appinfo; + } let data = { + sha: app.sha ? app.sha.toString('hex') : null, changenumber: app.change_number, missingToken: !!app.missing_token, - appinfo: VDF.parse(appInfoVdf).appinfo + appinfo }; if ((!cache.apps[app.appid] && requestType == PICSRequestType.Changelist) || (cache.apps[app.appid] && cache.apps[app.appid].changenumber != data.changenumber)) { @@ -248,12 +404,12 @@ class SteamUserApps extends SteamUserAppAuth { } cache.apps[app.appid] = data; - app._parsedData = data; }); (body.packages || []).forEach((pkg) => { let data = { + sha: pkg.sha ? pkg.sha.toString('hex') : null, changenumber: pkg.change_number, missingToken: !!pkg.missing_token, packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null @@ -264,33 +420,16 @@ class SteamUserApps extends SteamUserAppAuth { } cache.packages[pkg.packageid] = data; - pkg._parsedData = data; - // Request info for all the apps in this package, if this request didn't originate from the license list + // Request info for all the apps in this package, if this request didn't originate from the license list, because then we'll first process all packages before requesting all package contents if (requestType != PICSRequestType.Licenses) { let appids = (pkg.packageinfo || {}).appids || []; - this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => {}); + this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => { }); } }); } - (body.unknown_appids || []).forEach((appid) => { - response.unknownApps.push(appid); - let index = appids.indexOf(appid); - if (index != -1) { - appids.splice(index, 1); - } - }); - - (body.unknown_packageids || []).forEach((packageid) => { - response.unknownPackages.push(packageid); - let index = packageids.indexOf(packageid); - if (index != -1) { - packageids.splice(index, 1); - } - }); - (body.apps || []).forEach((app) => { // _parsedData will be populated if we have the PICS cache enabled. // If we don't, we need to parse the data here. @@ -301,9 +440,10 @@ class SteamUserApps extends SteamUserAppAuth { appInfoVdf = appInfoVdf.replace(/\0$/, ''); response.apps[app.appid] = app._parsedData || { - changenumber: app.change_number, - missingToken: !!app.missing_token, - appinfo: VDF.parse(appInfoVdf).appinfo + "sha": app.sha ? app.sha.toString('hex') : null, + "changenumber": app.change_number, + "missingToken": !!app.missing_token, + "appinfo": VDF.parse(appInfoVdf).appinfo }; let index = appids.indexOf(app.appid); @@ -314,9 +454,10 @@ class SteamUserApps extends SteamUserAppAuth { (body.packages || []).forEach((pkg) => { response.packages[pkg.packageid] = pkg._parsedData || { - changenumber: pkg.change_number, - missingToken: !!pkg.missing_token, - packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null + "sha": pkg.sha ? pkg.sha.toString('hex') : null, + "changenumber": pkg.change_number, + "missingToken": !!pkg.missing_token, + "packageinfo": pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null }; let index = packageids.indexOf(pkg.packageid); @@ -327,74 +468,177 @@ class SteamUserApps extends SteamUserAppAuth { // appids and packageids contain the list of IDs that we're still waiting on data for if (appids.length === 0 && packageids.length === 0) { - if (!inclTokens) { - return resolve(response); + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages } + return resolve(combined); + } + }; - // We want tokens - let tokenlessAppids = []; - let tokenlessPackages = []; + // If we don't use PICS cache or require fresh product info for changelist request, then just perform a normal request + if (!this.options.enablePicsCache || requestType == PICSRequestType.Changelist) { + return this._send(EMsg.ClientPICSProductInfoRequest, { apps, packages }, onResponse); + } - for (let appid in response.apps) { - if (response.apps[appid].missingToken) { - tokenlessAppids.push(parseInt(appid, 10)); - } - } + cached = await this._getCachedProductInfo(apps, packages); + // Note: This callback can be called multiple times + this._send(EMsg.ClientPICSProductInfoRequest, { apps, packages, meta_data_only: true }, async (body) => { + (body.apps || []).forEach((app) => { + shaList.apps[app.appid] = app.sha ? app.sha.toString('hex') : null; + }); - for (let packageid in response.packages) { - if (response.packages[packageid].missingToken) { - tokenlessPackages.push(parseInt(packageid, 10)); + (body.packages || []).forEach((pkg) => { + shaList.packages[pkg.packageid] = pkg.sha ? pkg.sha.toString('hex') : null; + }); + + response.unknownApps = response.unknownApps.concat(body.unknown_appids || []); + response.unknownPackages = response.unknownPackages.concat(body.unknown_packageids || []); + + let appTotal = Object.keys(shaList.apps).length + response.unknownApps.length; + let packageTotal = Object.keys(shaList.packages).length + response.unknownPackages.length; + + // If our collected totals match the (unique) total we requested + if (appTotal === appids.length && packageTotal === packageids.length) { + // Filter out any apps & packages we already have cached and do not need to be refreshed + apps = apps.filter((app) => !response.unknownApps.includes(app.appid) && (cached.apps[app.appid] || {}).sha !== shaList.apps[app.appid]); + packages = packages.filter((pkg) => !response.unknownPackages.includes(pkg.packageid) && (cached.packages[pkg.packageid] || {}).sha !== shaList.packages[pkg.packageid]); + // console.log( + // "requestType:", requestType, + // "app request:", appids.length, + // "pkg request:", packageids.length, + // "app unknown:", response.unknownApps.length, + // "pkg unknown:", response.unknownPackages.length, + // "app cache:", Object.keys(cached.apps).length, + // "pkg cache:", Object.keys(cached.packages).length, + // "app to refresh:", apps.length, + // "pkg to refresh:", packages.length, + // ); + + // If we have nothing to refresh / no stale data (e.g. all of them were unknown) + if (apps.length === 0 && packages.length === 0) { + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages } + return resolve(combined); } - if (tokenlessAppids.length == 0 && tokenlessPackages.length == 0) { - // No tokens needed - return resolve(response); + // We want tokens + if (inclTokens) { + let _appids = apps.map(app => app.appid); + let _packageids = packages.map(pkg => pkg.packageid); + let tokenlessAppids = body.apps.filter(app => _appids.includes(app.appid) && !!app.missing_token).map(app => app.appid); + let tokenlessPackages = body.packages.filter(pkg => _packageids.includes(pkg.packageid) && !!pkg.missing_token).map(pkg => pkg.packageid); + if (tokenlessAppids.length > 0 || tokenlessPackages.length > 0) { + try { + let { + appTokens, + packageTokens + } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); + let tokenApps = {}; + let tokenPackages = {}; + + for (let appid in appTokens) { + tokenApps[appid] = { + appid: parseInt(appid, 10), + access_token: appTokens[appid] + }; + } + + for (let packageid in packageTokens) { + tokenPackages[packageid] = { + packageid: parseInt(packageid, 10), + access_token: packageTokens[packageid] + }; + } + + // Replace products to request with included tokens + apps = apps.filter(app => !appTokens[app.appid]).concat(Object.values(tokenApps)); + packages = packages.filter(pkg => !tokenPackages[pkg.packageid]).concat(Object.values(tokenPackages)); + } catch (ex) { + return reject(ex); + } + } } - try { - let { - appTokens, - packageTokens - } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); - let tokenApps = []; - let tokenPackages = []; - - for (let appid in appTokens) { - tokenApps.push({appid: parseInt(appid, 10), access_token: appTokens[appid]}); - } + appids = apps.map(app => app.appid); + packageids = packages.map(pkg => pkg.packageid); - for (let packageid in packageTokens) { - tokenPackages.push({ - packageid: parseInt(packageid, 10), - access_token: packageTokens[packageid] - }); - } + // Request the apps & packages we need to refresh + this._send(EMsg.ClientPICSProductInfoRequest, { apps, packages }, onResponse); + } + }); + }); + } - // Now we have the tokens. Request the data. - let {apps, packages} = await this.getProductInfo(tokenApps, tokenPackages, false); - for (let appid in apps) { - response.apps[appid] = apps[appid]; - let index = response.unknownApps.indexOf(parseInt(appid, 10)); - if (index != -1) { - response.unknownApps.splice(index, 1); - } - } + /** + * Get info about some apps and/or packages from Steam, but first split it into chunks + * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} + * @param {int[]|object[]} packages - Array of package IDs. May be empty. May also contain objects with keys {packageid, access_token} + * @param {boolean} [inclTokens=false] - If true, automatically retrieve access tokens if needed + * @param {function} [callback] + * @param {int} [requestType] - Don't touch + * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} + */ + getProductInfo(apps, packages, inclTokens, callback, requestType) { + // Adds support for the previous syntax + if (typeof inclTokens !== 'boolean' && typeof inclTokens === 'function') { + requestType = callback; + callback = inclTokens; + inclTokens = false; + } - for (let packageid in packages) { - response.packages[packageid] = packages[packageid]; - let index = response.unknownPackages.indexOf(parseInt(packageid, 10)); - if (index != -1) { - response.unknownPackages.splice(index, 1); - } - } + // Add support for optional callback + if (!requestType && typeof callback !== 'function') { + requestType = callback; + callback = undefined; + } - resolve(response); - } catch (ex) { - return reject(ex); + requestType = requestType || PICSRequestType.User; + + // This one actually can take a while, so allow it to go as long as 120 minutes + return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { + try { + let response = { + apps: {}, + packages: {}, + unknownApps: [], + unknownPackages: [] + }; + // Split apps + packages into chunks of 2000 + let chunkSize = 2000; + for (let i = 0; i < packages.length; i += chunkSize) { + let packagesChunk = packages.slice(i, i + chunkSize); + // Do not include callback in the request, it will be called multiple times + let result = await this._getProductInfo([], packagesChunk, inclTokens, null, requestType); + response = { + apps: Object.assign(response.apps, result.apps), + packages: Object.assign(response.packages, result.packages), + unknownApps: response.unknownApps.concat(result.unknownApps), + unknownPackages: response.unknownPackages.concat(result.unknownPackages) } } - }); + for (let i = 0; i < apps.length; i += chunkSize) { + let appsChunk = apps.slice(i, i + chunkSize); + let result = await this._getProductInfo(appsChunk, [], inclTokens, null, requestType); + response = { + apps: Object.assign(response.apps, result.apps), + packages: Object.assign(response.packages, result.packages), + unknownApps: response.unknownApps.concat(result.unknownApps), + unknownPackages: response.unknownPackages.concat(result.unknownPackages) + } + } + resolve(response); + } catch (ex) { + return reject(ex); + } }); } @@ -469,6 +713,12 @@ class SteamUserApps extends SteamUserAppAuth { return; } + // First wait for ownership cache to be loaded, so we can calculate ourApps & ourPackages correctly. + if (!this.picsCache.ownershipModified) { + this._resetChangelistUpdateTimer(); + return; + } + let result = null; try { result = await this.getProductChanges(this.picsCache.changenumber); @@ -479,7 +729,7 @@ class SteamUserApps extends SteamUserAppAuth { } let cache = this.picsCache; - let {appChanges, packageChanges, currentChangeNumber} = result; + let { appChanges, packageChanges, currentChangeNumber } = result; cache.apps = cache.apps || {}; cache.packages = cache.packages || {}; @@ -498,6 +748,9 @@ class SteamUserApps extends SteamUserAppAuth { } cache.changenumber = currentChangeNumber; + if (this.options.savePicsCache) { + this._saveFile('changenumber.txt', currentChangeNumber); + } this._resetChangelistUpdateTimer(); return; } @@ -513,8 +766,11 @@ class SteamUserApps extends SteamUserAppAuth { this.emit('changelist', currentChangeNumber, appChanges, packageChanges); - let {appTokens, packageTokens} = result; + let { appTokens, packageTokens } = result; cache.changenumber = currentChangeNumber; + if (this.options.savePicsCache) { + this._saveFile('changenumber.txt', currentChangeNumber); + } this._resetChangelistUpdateTimer(); let index = -1; @@ -531,8 +787,22 @@ class SteamUserApps extends SteamUserAppAuth { } // Add a no-op catch in case there's some kind of error - this.getProductInfo(ourApps, ourPackages, false, null, PICSRequestType.Changelist).catch(() => { + let { packages } = await this.getProductInfo(ourApps, ourPackages, false, null, PICSRequestType.Changelist).catch(() => { }); + + // Request info for all the apps in these packages + let appids = []; + + for (let pkgid in packages) { + ((packages[pkgid].packageinfo || {}).appids || []).filter(appid => !appids.includes(appid)).forEach(appid => appids.push(appid)); + } + + try { + // Request type is changelist, because we need to refresh their pics cache + await this.getProductInfo(appids, [], true, undefined, PICSRequestType.Changelist); + } catch (ex) { + this.emit('debug', `Error retrieving product info of changed apps: ${ex.message}`); + } } /** @@ -549,7 +819,7 @@ class SteamUserApps extends SteamUserAppAuth { return; } - this.getProductInfo([appid], [], false, null, PICSRequestType.AddToCache).catch(() => { + this.getProductInfo([appid], [], true, null, PICSRequestType.AddToCache).catch(() => { }); } @@ -588,8 +858,8 @@ class SteamUserApps extends SteamUserAppAuth { return; } - let {packages} = result; - // Request info for all the apps in these packages + let { packages } = result; + // Request info for all the apps in these packages, and only after all packages have been processed let appids = []; for (let pkgid in packages) { @@ -630,14 +900,15 @@ class SteamUserApps extends SteamUserAppAuth { let appids = {}; ownedPackages.forEach((pkg) => { - if (!this.picsCache.packages[pkg]) { - this._warn(`Failed to get owned apps for package ${pkg}`); + let pkgid = pkg; + if (!this.picsCache.packages[pkgid]) { + this._warn(`Failed to get owned apps for package ${pkgid}: not in cache`); return; } - pkg = this.picsCache.packages[pkg]; + pkg = this.picsCache.packages[pkgid]; if (!pkg.packageinfo) { - this._warn(`Failed to get owned apps for package ${pkg}`); + this._warn(`Failed to get owned apps for package ${pkgid}: no package info`); return; } @@ -678,19 +949,20 @@ class SteamUserApps extends SteamUserAppAuth { let depotids = {}; ownedPackages.forEach((pkg) => { - if (!this.picsCache.packages[pkg]) { - this._warn(`Failed to get owned depots for package ${pkg}`); + let pkgid = pkg; + if (!this.picsCache.packages[pkgid]) { + this._warn(`Failed to get owned depots for package ${pkgid}: not in cache`); return; } - pkg = this.picsCache.packages[pkg]; + pkg = this.picsCache.packages[pkgid]; if (!pkg.packageinfo) { - this._warn(`Failed to get owned depots for package ${pkg}`); + this._warn(`Failed to get owned depots for package ${pkgid}: no package info`); return; } pkg = pkg.packageinfo; - (pkg.depotids || []).forEach(function(depotid) { + (pkg.depotids || []).forEach(function (depotid) { if (!depotids[depotid]) { depotids[depotid] = true; } @@ -757,7 +1029,7 @@ class SteamUserApps extends SteamUserAppAuth { } // If filter options is a boolean, we asssume it's excludeSharedLicenses if (typeof filter === 'boolean') { - filter = {excludeShared: filter}; + filter = { excludeShared: filter }; } // New behavior from this point on @@ -924,7 +1196,7 @@ class SteamUserApps extends SteamUserAppAuth { } return StdLib.Promises.timeoutCallbackPromise(10000, ['grantedPackageIds', 'grantedAppIds'], callback, (resolve, reject) => { - this._send(EMsg.ClientRequestFreeLicense, {appids: appIDs}, (body) => { + this._send(EMsg.ClientRequestFreeLicense, { appids: appIDs }, (body) => { if (body.eresult != EResult.OK) { reject(Helpers.eresultError(body.eresult)); } else { @@ -972,26 +1244,24 @@ class SteamUserApps extends SteamUserAppAuth { return reject(new Error(`Incorrect key length: expected ${keyLength - 1} but got ${key.length}`)); } - return resolve({key}); + return resolve({ key }); }); }); } } -SteamUserBase.prototype._handlerManager.add(EMsg.ClientLicenseList, function(body) { +SteamUserBase.prototype._handlerManager.add(EMsg.ClientLicenseList, function (body) { this.emit('licenses', body.licenses); this.licenses = body.licenses; // Request info for our licenses - if (this.options.enablePicsCache) { - this._getLicenseInfo(); - } + this._getLicenseInfo(); }); -SteamUserBase.prototype._handlerManager.add(EMsg.ClientPlayingSessionState, function(body) { +SteamUserBase.prototype._handlerManager.add(EMsg.ClientPlayingSessionState, function (body) { this._playingBlocked = body.playing_blocked; this.emit('playingState', body.playing_blocked, body.playing_app); - this.playingState = {blocked: body.playing_blocked, appid: body.playing_app}; + this.playingState = { blocked: body.playing_blocked, appid: body.playing_app }; }); function sortNumeric(a, b) { diff --git a/index.js b/index.js index 68e9d045..4c0bc4ac 100644 --- a/index.js +++ b/index.js @@ -86,6 +86,18 @@ class SteamUser extends SteamUserTwoFactor { this.storage = new FileManager(this.options.dataDirectory); } + if (this.options.savePicsCache) { + this._readFile('changenumber.txt').then((changenumber) => { + if (changenumber && isFinite(changenumber)) { + this.picsCache.changenumber = parseInt(changenumber, 10); + } + }); + } + + if (this.options.picsCacheAll && this.options.savePicsCache) { + this._warn('Both picsCacheAll and savePicsCache are enabled. Unless a custom storage engine is used, beware that this will cause a lot of disk IO and space usage!'); + } + this._initialized = true; } diff --git a/package.json b/package.json index 81e584a3..14ea9f4a 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "adm-zip": "^0.5.10", "binarykvparser": "^2.2.0", "bytebuffer": "^5.0.0", - "file-manager": "^2.0.0", + "file-manager": "^2.0.1", "kvparser": "^1.0.1", "lzma": "^2.3.2", "protobufjs": "^7.2.4", @@ -53,4 +53,4 @@ "engines": { "node": ">=14.0.0" } -} +} \ No newline at end of file diff --git a/resources/default_options.js b/resources/default_options.js index bdd51483..979e14ba 100644 --- a/resources/default_options.js +++ b/resources/default_options.js @@ -8,6 +8,7 @@ const EMachineIDType = require('./EMachineIDType.js'); * @property {EMachineIDType} [machineIdType] * @property {string[]} [machineIdFormat] * @property {boolean} [enablePicsCache=false] + * @property {boolean} [savePicsCache=false] * @property {boolean} [picsCacheAll=false] * @property {number} [changelistUpdateInterval=60000] * @property {PackageFilter|PackageFilterFunction|null} [ownershipFilter=null] @@ -28,6 +29,7 @@ module.exports = { machineIdType: EMachineIDType.AccountNameGenerated, machineIdFormat: ['SteamUser Hash BB3 {account_name}', 'SteamUser Hash FF2 {account_name}', 'SteamUser Hash 3B3 {account_name}'], enablePicsCache: false, + savePicsCache: false, picsCacheAll: false, changelistUpdateInterval: 60000, additionalHeaders: {},