From 3c0faa9217d3265c2eddaeca002e6e3a3366ceed Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Tue, 13 Feb 2024 11:51:35 -0500 Subject: [PATCH] feat: multithreaded downloads --- download.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/download.py b/download.py index 976e68d..627190e 100644 --- a/download.py +++ b/download.py @@ -3,10 +3,13 @@ import cgi import os import datetime +from multiprocessing.pool import ThreadPool from do_authentication import authenticate from do_http_get import do_get +MAX_THREADS = 6 + ############################################################################################################## # First Step: Get the config data from config.json file ############################################################################################################## @@ -142,9 +145,8 @@ def download_zone_files(urls, working_directory): if not os.path.exists(output_directory): os.makedirs(output_directory) - # Download the zone files one by one - for link in urls: - download_one_zone(link, output_directory) + pool = ThreadPool(MAX_THREADS) + pool.map(lambda link: download_one_zone(link, output_directory), urls) # Finally, download all zone files start_time = datetime.datetime.now()