Load installer metadata from Github Releases

- Try to fix "Error 429: Too Many Requests" when downloading from https://raw.githubusercontent.com/
- See Issue #236 for details
- Installer metadata is now hosted on the python-patcher-metadata repository as releases

Author: drojf

README.md

@@ -10,7 +10,7 @@ This repository contains the mod installer, used for installing all 07th-mod mod
 
 ### Updating landing page and mod page "update" banners
 
-The HTML for the landing page and update banners for each mod is dynamically fetched from this github repo [https://github.com/07th-mod/python-patcher-updates](https://github.com/07th-mod/python-patcher-updates).
+The HTML for the landing page and update banners for each mod is dynamically fetched from this github repo [https://github.com/07th-mod/python-patcher-metadata](https://github.com/07th-mod/python-patcher-metadata).
 
 To update the html:
 

common.py

@@ -127,7 +127,15 @@ def printErrorMessage(text):
 
 ################################################## Global Variables#####################################################
 class Globals:
-	GITHUB_MASTER_BASE_URL = "https://raw.githubusercontent.com/07th-mod/python-patcher/master/"
+	# On startup, the installer will download a zip file containing the following files
+	# and extract them to the INSTALLER_META_DATA_FOLDER
+	INSTALLER_META_DATA_ZIP_URL = "https://github.com/07th-mod/python-patcher-metadata/releases/latest/download/installerMetadata.zip"
+	INSTALLER_META_DATA_FOLDER = "installerMetadata"
+	META_PATH__INSTALL_DATA = os.path.join(INSTALLER_META_DATA_FOLDER, "installData.json")
+	META_PATH__VERSION_DATA = os.path.join(INSTALLER_META_DATA_FOLDER, "versionData.json")
+	META_PATH__DOWNLOAD_SIZES = os.path.join(INSTALLER_META_DATA_FOLDER, "cachedDownloadSizes.json")
+	META_PATH__UPDATES = os.path.join(INSTALLER_META_DATA_FOLDER, "updates.json")
+
 	# The installer info version this installer is compatibile with
 	# Increment it when you make breaking changes to the json files
 	JSON_VERSION = 12
@@ -377,7 +385,7 @@ def loadCachedDownloadSizes(modList):
 			if Globals.DEVELOPER_MODE:
 				downloadSizesDict, _error = getJSON('cachedDownloadSizes.json', isURL=False)
 			else:
-				downloadSizesDict, _error = getJSON(Globals.GITHUB_MASTER_BASE_URL + 'cachedDownloadSizes.json', isURL=True)
+				downloadSizesDict, _error = getJSON(Globals.META_PATH__DOWNLOAD_SIZES, isURL=False)
 
 			if downloadSizesDict is None:
 				print("ERROR: Failed to retrieve cachedDownloadSizes.json file")
@@ -749,7 +757,7 @@ def sevenZipTest(archive_path):
 	return runProcessOutputToTempFile(arguments, sevenZipMode=True)
 
 def preloadModUpdatesHTML():
-	html, errorInfo = getJSON("https://github.com/07th-mod/python-patcher-updates/releases/latest/download/updates.json", isURL=True)
+	html, errorInfo = getJSON(Globals.META_PATH__UPDATES, isURL=False)
 
 	if errorInfo is not None:
 		print('WARNING: Failed to get mod updates from server')
@@ -1483,6 +1491,7 @@ def downloadFile(url, is_text):
 	:return:
 	"""
 	def downloadUsingURLOpen(download_url):
+		print("Downloading [{}] using Python urlopen...".format(download_url))
 		file = urlopen(Request(download_url, headers={"User-Agent": ""}), context=Globals.getURLOpenContext())
 		data = file.read()
 		file.close()

fileVersionManagement.py

@@ -246,13 +246,11 @@ def getLocalVersion(localVersionFilePath):
 
 
 def getRemoteVersion(remoteTargetID):
-	remoteVersionURL = common.Globals.GITHUB_MASTER_BASE_URL + "versionData.json"
-
 	# Get remote version
 	if common.Globals.DEVELOPER_MODE and os.path.exists("versionData.json"):
 		allRemoteVersions, remoteError = common.getJSON("versionData.json", isURL=False)
 	else:
-		allRemoteVersions, remoteError = common.getJSON(remoteVersionURL, isURL=True)
+		allRemoteVersions, remoteError = common.getJSON(common.Globals.META_PATH__VERSION_DATA, isURL=False)
 
 	if remoteError is not None:
 		print("Error retrieving remote version".format(remoteError))

main.py

@@ -42,7 +42,7 @@ def getModList(is_developer=True):
 	if is_developer and os.path.exists('installData.json'):
 		return common.getModList("installData.json", isURL=False)
 	else:
-		return common.getModList(common.Globals.GITHUB_MASTER_BASE_URL + "installData.json", isURL=True)
+		return common.getModList(common.Globals.META_PATH__INSTALL_DATA, isURL=False)
 
 def getSubModConfigList(modList):
 	subModconfigList = []
@@ -207,6 +207,28 @@ def installerCommonStartupTasks():
 	installerGUI = httpGUI.InstallerGUI()
 
 	def thread_getSubModConfigList():
+		# Download and extract the installer meta data zip file, containing download links, file versions, file size etc.
+		# If you've cloned the repo (developer) then the installer will download these files, but won't use them except for
+		# the updates.json file, which is located in the other repo "python-patcher-status"
+
+		# Delete the old file to be sure we're using the latest metadata.
+		installerMetadataZipPath = "installerMetadata.zip"
+		common.removeFileWithCheck(installerMetadataZipPath)
+
+		# Download and extract the installer metadata zip
+		# Abort installation if we can't download/extract this file
+		url = common.Globals.INSTALLER_META_DATA_ZIP_URL
+		if common.aria(url=url, outputFile="installerMetadata.zip") != 0:
+			raise Exception("ERROR - could not download [{}] with aria2c".format(url))
+
+		# Extract the zip file. This should overwrite existing files so we don't get old metadata.
+		common.extractOrCopyFile(
+			filename = "installerMetadata.zip",
+			sourceFolder = ".",
+			destinationFolder = common.Globals.INSTALLER_META_DATA_FOLDER
+		)
+
+		# Now parse/load the downloaded files. Note that some files may be used later in the installer.
 		modList = getModList(common.Globals.DEVELOPER_MODE)
 		common.Globals.loadCachedDownloadSizes(modList)
 		return getSubModConfigList(modList)