From 785fbc45f4358a66d4c820d11b8100f1f00cc44a Mon Sep 17 00:00:00 2001 From: ben reed Date: Fri, 27 May 2022 15:25:56 -0700 Subject: [PATCH] add on_read2 to allow for better progress bar during download --- mosspy/download_report.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mosspy/download_report.py b/mosspy/download_report.py index 5040ee1..5fd5451 100644 --- a/mosspy/download_report.py +++ b/mosspy/download_report.py @@ -6,13 +6,14 @@ except ImportError: from urllib2 import urlopen -def process_url(url, urls, base_url, path, on_read): +def process_url(url, urls, base_url, path, on_read, on_read2): from bs4 import BeautifulSoup # Backward compability, don't break Moss when bs4 not available. logging.debug ("Processing URL: " + url) response = urlopen(url) html = response.read() on_read(url) + on_read2(url, len(urls)) soup = BeautifulSoup(html, 'lxml') file_name = os.path.basename(url) @@ -50,7 +51,7 @@ def process_url(url, urls, base_url, path, on_read): f.write(soup.encode(soup.original_encoding)) f.close() -def download_report(url, path, connections = 4, log_level=logging.DEBUG, on_read=lambda url: None): +def download_report(url, path, connections = 4, log_level=logging.DEBUG, on_read=lambda url: None, on_read2=lambda url, left: None): logging.basicConfig(level=log_level) if len(url) == 0: @@ -69,7 +70,7 @@ def download_report(url, path, connections = 4, log_level=logging.DEBUG, on_read # Handling thread for url in urls: - t = Thread(target=process_url, args=[url, urls, base_url, path, on_read]) + t = Thread(target=process_url, args=[url, urls, base_url, path, on_read, on_read2]) t.start() threads.append(t)