From dc5a6ac60460ac90dbafff1200e845dfed30e0a0 Mon Sep 17 00:00:00 2001 From: Joe Bloggs Date: Mon, 13 Apr 2020 19:07:42 +0100 Subject: [PATCH 1/8] Add download_folder config file option --- gplaycli/gplaycli.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/gplaycli/gplaycli.py b/gplaycli/gplaycli.py index 718906c..f2e11cb 100755 --- a/gplaycli/gplaycli.py +++ b/gplaycli/gplaycli.py @@ -104,16 +104,17 @@ def __init__(self, args=None, config_file=None): self.token_url = config.get('Credentials', 'token_url', fallback='https://matlink.fr/token/email/gsfid') self.keyring_service = config.get('Credentials', 'keyring_service', fallback=None) - self.tokencachefile = os.path.expanduser(config.get("Cache", "token", fallback="token.cache")) - self.yes = config.getboolean('Misc', 'accept_all', fallback=False) - self.verbose = config.getboolean('Misc', 'verbose', fallback=False) - self.append_version = config.getboolean('Misc', 'append_version', fallback=False) - self.progress_bar = config.getboolean('Misc', 'progress', fallback=False) - self.logging_enable = config.getboolean('Misc', 'enable_logging', fallback=False) - self.addfiles_enable = config.getboolean('Misc', 'enable_addfiles', fallback=False) - self.device_codename = config.get('Device', 'codename', fallback='bacon') - self.locale = config.get("Locale", "locale", fallback="en_GB") - self.timezone = config.get("Locale", "timezone", fallback="CEST") + self.tokencachefile = os.path.expanduser(config.get('Cache', 'token', fallback='token.cache')) + self.download_folder = os.path.expanduser(config.get('Cache', 'download_folder', fallback='.')) + self.yes = config.getboolean('Misc', 'accept_all', fallback=False) + self.verbose = config.getboolean('Misc', 'verbose', fallback=False) + self.append_version = config.getboolean('Misc', 'append_version', fallback=False) + self.progress_bar = config.getboolean('Misc', 'progress', fallback=False) + self.logging_enable = config.getboolean('Misc', 'enable_logging', fallback=False) + self.addfiles_enable = config.getboolean('Misc', 'enable_addfiles', fallback=False) + self.device_codename = config.get('Device', 'codename', fallback='bacon') + self.locale = config.get('Locale', 'locale', fallback='en_GB') + self.timezone = config.get('Locale', 'timezone', fallback='CEST') if not args: return @@ -138,6 +139,9 @@ def __init__(self, args=None, config_file=None): if args.update is not None: self.download_folder = args.update + if args.download is not None and args.folder is not None: + self.download_folder = args.folder[0] + if args.log is not None: self.logging_enable = args.log @@ -337,8 +341,8 @@ def search(self, search_string, free_only=True, include_headers=True): """ Search the given string search_string on the Play Store. - search_string -- the string to search on the Play Store - free_only -- True if only costless apps should be searched for + search_string -- the string to search on the Play Store + free_only -- True if only costless apps should be searched for include_headers -- True if the result table should show column names """ try: @@ -359,7 +363,7 @@ def search(self, search_string, free_only=True, include_headers=True): for app in cluster["child"]: # skip that app if it not free # or if it's beta (pre-registration) - if ('offer' not in app # beta apps (pre-registration) + if ('offer' not in app # beta apps (pre-registration) or free_only and app['offer'][0]['checkoutFlowRequired'] # not free to download ): @@ -618,7 +622,7 @@ def main(): parser.add_argument('-a', '--additional-files', help="Enable the download of additional files", action='store_true', default=False) parser.add_argument('-F', '--file', help="Load packages to download from file, one package per line", metavar="FILE") parser.add_argument('-u', '--update', help="Update all APKs in a given folder", metavar="FOLDER") - parser.add_argument('-f', '--folder', help="Where to put the downloaded Apks, only for -d command", metavar="FOLDER", nargs=1, default=['.']) + parser.add_argument('-f', '--folder', help="Where to put the downloaded Apks, only for -d command", metavar="FOLDER", nargs=1, default=None) parser.add_argument('-dc', '--device-codename', help="The device codename to fake", choices=GooglePlayAPI.getDevicesCodenames(), metavar="DEVICE_CODENAME") parser.add_argument('-t', '--token', help="Instead of classical credentials, use the tokenize version", action='store_true', default=None) parser.add_argument('-tu', '--token-url', help="Use the given tokendispenser URL to retrieve a token", metavar="TOKEN_URL") @@ -654,8 +658,6 @@ def main(): args.download = util.load_from_file(args.file) if args.download is not None: - if args.folder is not None: - cli.download_folder = args.folder[0] cli.download(args.download) From cb73084fef12966876744c09d973afd204ef924b Mon Sep 17 00:00:00 2001 From: Joe Bloggs Date: Tue, 14 Apr 2020 15:58:30 +0100 Subject: [PATCH 2/8] Put log files in downloads folder --- gplaycli/gplaycli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gplaycli/gplaycli.py b/gplaycli/gplaycli.py index f2e11cb..1c13845 100755 --- a/gplaycli/gplaycli.py +++ b/gplaycli/gplaycli.py @@ -167,9 +167,9 @@ def __init__(self, args=None, config_file=None): raise TypeError("Token string and GSFID have to be passed at the same time.") if self.logging_enable: - self.success_logfile = "apps_downloaded.log" - self.failed_logfile = "apps_failed.log" - self.unavail_logfile = "apps_not_available.log" + self.success_logfile = os.path.join(self.download_folder,"apps_downloaded.log") + self.failed_logfile = os.path.join(self.download_folder,"apps_failed.log") + self.unavail_logfile = os.path.join(self.download_folder,"apps_not_available.log") ########## Public methods ########## From af948545676919818050e44364778913be2268f1 Mon Sep 17 00:00:00 2001 From: Joe Bloggs Date: Tue, 14 Apr 2020 23:43:16 +0100 Subject: [PATCH 3/8] Ignore comment lines in package downloads file Its useful to be able to have comment lines to explain the rationale for a particular set of apps. --- gplaycli/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gplaycli/util.py b/gplaycli/util.py index 26f238e..a42d41d 100644 --- a/gplaycli/util.py +++ b/gplaycli/util.py @@ -10,7 +10,8 @@ def sizeof_fmt(num): return "%.2f%s" % (num/(1024**log), ['B ','KB','MB','GB','TB'][log]) def load_from_file(filename): - return [package.strip('\r\n') for package in open(filename).readlines()] + return [package.strip('\r\n') for package in open(filename).readlines() + if not package.startswith('#')] def list_folder_apks(folder): """ From bb58a1494f11eb6d2fa9287fa620adf0aef6fbc6 Mon Sep 17 00:00:00 2001 From: Joe Bloggs Date: Fri, 17 Apr 2020 00:20:07 +0100 Subject: [PATCH 4/8] Put split files in download folder --- gplaycli/gplaycli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gplaycli/gplaycli.py b/gplaycli/gplaycli.py index 1c13845..0182a7c 100755 --- a/gplaycli/gplaycli.py +++ b/gplaycli/gplaycli.py @@ -317,7 +317,8 @@ def download(self, pkg_todownload): for split in splits: split_total_size = int(split['file']['total_size']) split_chunk_size = int(split['file']['chunk_size']) - with open(split['name'], "wb") as fbuffer: + split_filename = os.path.join(download_folder, split['name']) + with open(split_filename, "wb") as fbuffer: bar = util.progressbar(expected_size=split_total_size, hide=not self.progress_bar) for index, chunk in enumerate(split["file"]["data"]): fbuffer.write(chunk) From 81b0d4263a2b79517e7eaac516a3bef5f91a587f Mon Sep 17 00:00:00 2001 From: Joe Bloggs Date: Fri, 17 Apr 2020 02:28:24 +0100 Subject: [PATCH 5/8] Create separate directory for split apks & apks with obb files --- gplaycli/gplaycli.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/gplaycli/gplaycli.py b/gplaycli/gplaycli.py index 0182a7c..5a5f01c 100755 --- a/gplaycli/gplaycli.py +++ b/gplaycli/gplaycli.py @@ -17,6 +17,7 @@ """ import os +import re import sys import site import json @@ -283,6 +284,12 @@ def download(self, pkg_todownload): failed_downloads.append((item, exc)) continue + additional_data = data_iter['additionalData'] + splits = data_iter['splits'] + if(additional_data or splits): + download_folder = os.path.join(download_folder, re.sub(".apk$","",filename)) + os.mkdir(download_folder) + filepath = os.path.join(download_folder, filename) #if file exists, continue @@ -290,8 +297,6 @@ def download(self, pkg_todownload): logger.info("File %s already exists, skipping.", filename) continue - additional_data = data_iter['additionalData'] - splits = data_iter['splits'] total_size = int(data_iter['file']['total_size']) chunk_size = int(data_iter['file']['chunk_size']) try: @@ -317,7 +322,7 @@ def download(self, pkg_todownload): for split in splits: split_total_size = int(split['file']['total_size']) split_chunk_size = int(split['file']['chunk_size']) - split_filename = os.path.join(download_folder, split['name']) + split_filename = os.path.join(download_folder, split['name']) + ".apk" with open(split_filename, "wb") as fbuffer: bar = util.progressbar(expected_size=split_total_size, hide=not self.progress_bar) for index, chunk in enumerate(split["file"]["data"]): From 3e736feba95260bdc55a11c18bb7956f670df1cd Mon Sep 17 00:00:00 2001 From: Joe Bloggs Date: Tue, 28 Apr 2020 19:57:14 +0100 Subject: [PATCH 6/8] Fixes to make sure updates work with split apks in folders --- gplaycli/gplaycli.py | 21 +++++++++++++-------- gplaycli/util.py | 2 ++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/gplaycli/gplaycli.py b/gplaycli/gplaycli.py index 5a5f01c..286b91c 100755 --- a/gplaycli/gplaycli.py +++ b/gplaycli/gplaycli.py @@ -262,6 +262,8 @@ def download(self, pkg_todownload): filename = "%s-v.%s.apk" % (detail['docid'], detail['details']['appDetails']['versionString']) else: filename = "%s.apk" % detail['docid'] + else: + filename = os.path.basename(filename) logger.info("%s / %s %s", 1+position, len(pkg_todownload), packagename) @@ -286,11 +288,13 @@ def download(self, pkg_todownload): additional_data = data_iter['additionalData'] splits = data_iter['splits'] + app_download_folder = download_folder if(additional_data or splits): - download_folder = os.path.join(download_folder, re.sub(".apk$","",filename)) - os.mkdir(download_folder) - - filepath = os.path.join(download_folder, filename) + app_download_folder = os.path.join(download_folder, re.sub(".apk$","",filename)) + if not os.path.exists(app_download_folder): + os.mkdir(app_download_folder) + + filepath = os.path.join(app_download_folder, filename) #if file exists, continue if self.append_version and os.path.isfile(filepath): @@ -309,7 +313,7 @@ def download(self, pkg_todownload): if additional_data: for obb_file in additional_data: obb_filename = "%s.%s.%s.obb" % (obb_file["type"], obb_file["versionCode"], data_iter["docId"]) - obb_filename = os.path.join(download_folder, obb_filename) + obb_filename = os.path.join(app_download_folder, obb_filename) obb_total_size = int(obb_file['file']['total_size']) obb_chunk_size = int(obb_file['file']['chunk_size']) with open(obb_filename, "wb") as fbuffer: @@ -322,7 +326,7 @@ def download(self, pkg_todownload): for split in splits: split_total_size = int(split['file']['total_size']) split_chunk_size = int(split['file']['chunk_size']) - split_filename = os.path.join(download_folder, split['name']) + ".apk" + split_filename = os.path.join(app_download_folder, split['name']) + ".apk" with open(split_filename, "wb") as fbuffer: bar = util.progressbar(expected_size=split_total_size, hide=not self.progress_bar) for index, chunk in enumerate(split["file"]["data"]): @@ -525,8 +529,9 @@ def analyse_local_apks(self, list_of_apks, download_folder): logger.info("Analyzing %s", filepath) apk = APK(filepath) packagename = apk.package - package_bunch.append(packagename) - version_codes.append(util.vcode(apk.version_code)) + if not packagename in package_bunch: + package_bunch.append(packagename) + version_codes.append(util.vcode(apk.version_code)) # BulkDetails requires only one HTTP request # Get APK info from store diff --git a/gplaycli/util.py b/gplaycli/util.py index a42d41d..e3addc1 100644 --- a/gplaycli/util.py +++ b/gplaycli/util.py @@ -18,6 +18,8 @@ def list_folder_apks(folder): List apks in the given folder """ list_of_apks = [filename for filename in os.listdir(folder) if filename.endswith(".apk")] + list_of_apks += [os.path.join(d,d+".apk") for d in os.listdir(folder) + if os.path.isfile(os.path.join(folder,d,d+".apk"))] return list_of_apks def vcode(string_vcode): From 6c63bdde8cc291d502af45755b055347a09aabaa Mon Sep 17 00:00:00 2001 From: Joe Bloggs Date: Thu, 30 Apr 2020 15:32:58 +0100 Subject: [PATCH 7/8] Make sure version numbers are updated on filenames --- gplaycli/gplaycli.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/gplaycli/gplaycli.py b/gplaycli/gplaycli.py index 286b91c..ef213f7 100755 --- a/gplaycli/gplaycli.py +++ b/gplaycli/gplaycli.py @@ -258,12 +258,14 @@ def download(self, pkg_todownload): packagename, filename = item if filename is None: - if self.append_version: - filename = "%s-v.%s.apk" % (detail['docid'], detail['details']['appDetails']['versionString']) - else: - filename = "%s.apk" % detail['docid'] + filename = detail['docid'] + else: + filename = re.sub('-v[0-9.-]+(apk)?','',os.path.basename(filename)) + + if self.append_version: + filename = "%s-v.%s.apk" % (filename, detail['details']['appDetails']['versionString']) else: - filename = os.path.basename(filename) + filename = "%s.apk" % filename logger.info("%s / %s %s", 1+position, len(pkg_todownload), packagename) From e97e089998ea974d8611a14b9c83a23f302aa419 Mon Sep 17 00:00:00 2001 From: Joe Bloggs Date: Fri, 1 May 2020 03:24:34 +0100 Subject: [PATCH 8/8] Bugfix: make sure filenames match package names --- gplaycli/gplaycli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gplaycli/gplaycli.py b/gplaycli/gplaycli.py index ef213f7..9fff8c4 100755 --- a/gplaycli/gplaycli.py +++ b/gplaycli/gplaycli.py @@ -522,6 +522,7 @@ def analyse_local_apks(self, list_of_apks, download_folder): in the download_folder folder. """ list_apks_to_update = [] + list_of_uniq_apks = [] package_bunch = [] version_codes = [] unavail_items = [] @@ -532,13 +533,14 @@ def analyse_local_apks(self, list_of_apks, download_folder): apk = APK(filepath) packagename = apk.package if not packagename in package_bunch: + list_of_uniq_apks.append(filepath) package_bunch.append(packagename) version_codes.append(util.vcode(apk.version_code)) # BulkDetails requires only one HTTP request # Get APK info from store details = self.api.bulkDetails(package_bunch) - for detail, packagename, filename, apk_version_code in zip(details, package_bunch, list_of_apks, version_codes): + for detail, packagename, filename, apk_version_code in zip(details, package_bunch, list_of_uniq_apks, version_codes): # this app is not in the play store if not detail: unavail_items.append(((packagename, filename), UNAVAIL))