From 2f22a25e34bb801b991cbc24693ed5c64cb0ac88 Mon Sep 17 00:00:00 2001 From: Rob Kam Date: Sun, 22 Oct 2023 18:52:35 +0100 Subject: [PATCH 1/2] Slightly update gui.py --- wikiteam/gui.py | 136 ++++++++++++++++++++++-------------------------- 1 file changed, 63 insertions(+), 73 deletions(-) diff --git a/wikiteam/gui.py b/wikiteam/gui.py index 61ae107..b3f8ec7 100644 --- a/wikiteam/gui.py +++ b/wikiteam/gui.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (C) 2011-2012 WikiTeam +# Copyright (C) 2011-2023 WikiTeam developers and MediaWiki Client Tools # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -22,6 +22,7 @@ * advanced: batch downloads, upload to Internet Archive or anywhere """ + import os import platform import random @@ -54,25 +55,27 @@ from wikiteam3.dumpgenerator.api.api import checkAPI from wikiteam3.dumpgenerator.api.index_check import checkIndex +# See https://www.mediawiki.org/wiki/Hosting_services wikifarms = { - "gentoo_wikicom": "Gentoo Wiki", - "opensuseorg": "OpenSuSE", - "referatacom": "Referata", - "shoutwikicom": "ShoutWiki", - "Unknown": "Unknown", - "wikanda": "Wikanda", - "wikifur": "WikiFur", - "wikimedia": "Wikimedia", - "wikitravelorg": "WikiTravel", - "wikkii": "Wikkii", + "fandom": "Fandom", + "miraheze": "Miraheze", + "mywikis": "MyWikis", + "neoseeker": "Neoseeker", + "prowiki": "ProWiki", + "shoutwiki": "ShoutWiki", + "telepedia wiki": "Telepedia Wiki", + "webmo": "WebMo", + "wiki.gg": "wiki.gg", + "wiki-site": "Wiki-Site", + "wikiforge": "WikiForge", + "wikitide": "WikiTide", } -NAME = "WikiTeam tools" +NAME = "MediaWiki Dump Generator" VERSION = "0.1" -HOMEPAGE = "https://code.google.com/p/wikiteam/" +HOMEPAGE = "https://github.com/mediawiki-client-tools/mediawiki-dump-generator" LINUX = platform.system().lower() == "linux" -PATH = os.path.dirname(__file__) -if PATH: +if PATH := os.path.dirname(__file__): os.chdir(PATH) @@ -177,16 +180,18 @@ def __init__(self, master): self.frame2, self.optionmenu21var, self.optionmenu21var.get(), - "Gentoo Wiki", - "OpenSuSE", - "Referata", + "Fandom", + "Miraheze", + "MyWikis", + "Neoseeker", + "ProWiki", "ShoutWiki", - "Unknown", - "Wikanda", - "WikiFur", - "Wikimedia", - "WikiTravel", - "Wikkii", + "Telepedia Wiki", + "WebMo", + "wiki.gg", + "Wiki-Site", + "WikiForge", + "WikiTide", ) self.optionmenu21.grid(row=1, column=1) @@ -215,6 +220,18 @@ def __init__(self, master): self.optionmenu23var.get(), "2011", "2012", + "2013", + "2014", + "2015", + "2016", + "2017", + "2018", + "2019", + "2020", + "2021", + "2022", + "2023", + "2024", ) self.optionmenu23.grid(row=1, column=5) @@ -228,7 +245,6 @@ def __init__(self, master): self.optionmenu24var.get(), "Google Code", "Internet Archive", - "ScottDB", ) self.optionmenu24.grid(row=1, column=7) @@ -367,14 +383,12 @@ def sumSizes(self, sizes): total += float(size.split(" ")[0]) * 1024 * 1024 elif size.endswith("TB"): total += float(size.split(" ")[0]) * 1024 * 1024 * 1024 - elif not size or size.lower() == "unknown": - pass - else: + elif size and size.lower() != "unknown": total += size return total / 1024 # MB def run(self): - for i in range(10): + for _ in range(10): time.sleep(0.1) self.value += 10 @@ -416,13 +430,11 @@ def downloadProgress(self, block_count, block_size, total_size): percent = downloaded / (total_mb / 100.0) if not random.randint(0, 10): msg = "{:.1f} MB of {:.1f} MB downloaded ({:.1f}%)".format( - downloaded, - total_mb, - percent if percent <= 100 else 100, + downloaded, total_mb, min(percent, 100) ) self.msg(msg, level="ok") - # sys.stdout.write("%.1f MB of %.1f MB downloaded (%.2f%%)" %(downloaded, total_mb, percent)) - # sys.stdout.flush() + # sys.stdout.write("%.1f MB of %.1f MB downloaded (%.2f%%)" %(downloaded, total_mb, percent)) + # sys.stdout.flush() except: pass @@ -432,15 +444,14 @@ def downloadDump(self, event=None): return else: self.block = True - items = self.tree.selection() - if items: + if items := self.tree.selection(): if not os.path.exists(self.downloadpath): os.makedirs(self.downloadpath) c = 0 d = 0 for item in items: filepath = ( - self.downloadpath + "/" + self.dumps[int(item)][0] + f"{self.downloadpath}/{self.dumps[int(item)][0]}" if self.downloadpath else self.dumps[int(item)][0] ) @@ -462,10 +473,7 @@ def downloadDump(self, event=None): filepath, reporthook=self.downloadProgress, ) - msg = "{} size is {} bytes large. Download successful!".format( - self.dumps[int(item)][0], - os.path.getsize(filepath), - ) + msg = f"{self.dumps[int(item)][0]} size is {os.path.getsize(filepath)} bytes large. Download successful!" self.msg(msg=msg, level="ok") c += 1 self.dumps[int(item)] = self.dumps[int(item)][:6] + ["True"] @@ -503,8 +511,9 @@ def clearAvailableDumps(self): self.tree.delete(str(i)) def showAvailableDumps(self): - c = 0 - for filename, wikifarm, size, date, mirror, url, downloaded in self.dumps: + for c, (filename, wikifarm, size, date, mirror, url, downloaded) in enumerate( + self.dumps + ): self.tree.insert( "", "end", @@ -520,7 +529,6 @@ def showAvailableDumps(self): ), tags=("downloaded" if downloaded else "nodownloaded",), ) - c += 1 def filterAvailableDumps(self): self.clearAvailableDumps() @@ -541,22 +549,13 @@ def filterAvailableDumps(self): else: nodownloadedsizes.append(self.dumps[i][2]) elif ( - ( - self.optionmenu21var.get() != "all" - and not self.optionmenu21var.get() == self.dumps[i][1] - ) - or ( - self.optionmenu22var.get() != "all" - and not self.optionmenu22var.get() in self.dumps[i][2] - ) - or ( - self.optionmenu23var.get() != "all" - and not self.optionmenu23var.get() in self.dumps[i][3] - ) - or ( - self.optionmenu24var.get() != "all" - and not self.optionmenu24var.get() in self.dumps[i][4] - ) + self.optionmenu21var.get() not in ["all", self.dumps[i][1]] + or self.optionmenu22var.get() != "all" + and self.optionmenu22var.get() not in self.dumps[i][2] + or self.optionmenu23var.get() != "all" + and self.optionmenu23var.get() not in self.dumps[i][3] + or self.optionmenu24var.get() != "all" + and self.optionmenu24var.get() not in self.dumps[i][4] ): self.tree.detach(str(i)) # hide this item sizes.append(self.dumps[i][2]) @@ -580,7 +579,7 @@ def isDumpDownloaded(self, filename): # improve, size check or md5sum? if filename: filepath = ( - self.downloadpath + "/" + filename if self.downloadpath else filename + f"{self.downloadpath}/{filename}" if self.downloadpath else filename ) if os.path.exists(filepath): return True @@ -619,18 +618,13 @@ def loadAvailableDumps(self): "http://www.archive.org/details/WikiTeamMirror", iaregexp, ], - [ - "ScottDB", - "http://mirrors.sdboyd56.com/WikiTeam/", - r'(?P[\d\.]+ (?:KB|MB|GB|TB))', - ], [ "Wikimedia", "http://dumps.wikimedia.org/backup-index.html", r'(?P)[^>]+: Dump complete', ], ] - wikifarms_r = re.compile(r"(%s)" % ("|".join(wikifarms.keys()))) + wikifarms_r = re.compile(f'({"|".join(wikifarms.keys())})') c = 0 for mirror, url, regexp in self.urls: print("Loading data from", mirror, url) @@ -640,9 +634,7 @@ def loadAvailableDumps(self): for i in m: filename = i.group("filename") if mirror == "Wikimedia": - filename = "%s-pages-meta-history.xml.7z" % ( - re.sub("/", "-", filename) - ) + filename = f'{re.sub("/", "-", filename)}-pages-meta-history.xml.7z' wikifarm = "Unknown" if re.search(wikifarms_r, filename): wikifarm = re.findall(wikifarms_r, filename)[0] @@ -658,13 +650,11 @@ def loadAvailableDumps(self): date = re.findall(r"\-(\d{4}\-\d{2}\-\d{2})[\.-]", filename)[0] downloadurl = "" if mirror == "Google Code": - downloadurl = "https://wikiteam.googlecode.com/files/" + filename + downloadurl = f"https://wikiteam.googlecode.com/files/{filename}" elif mirror == "Internet Archive": downloadurl = ( re.sub(r"/details/", r"/download/", url) + "/" + filename ) - elif mirror == "ScottDB": - downloadurl = url + "/" + filename elif mirror == "Wikimedia": downloadurl = ( "http://dumps.wikimedia.org/" From 0a13d94c7b379607560847bfb6d15f6601651dae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 22 Oct 2023 17:53:21 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- wikiteam/uploader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wikiteam/uploader.py b/wikiteam/uploader.py index c2e5480..256ae31 100644 --- a/wikiteam/uploader.py +++ b/wikiteam/uploader.py @@ -28,7 +28,6 @@ import requests from internetarchive import get_item - from wikiteam3.dumpgenerator.config import Config from wikiteam3.utils import domain2prefix, getUserAgent