From c3aceb22f8502ba7dcb02be7a08178c092ceab99 Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Wed, 15 Jan 2025 01:42:45 -0500 Subject: [PATCH 1/4] Switch from imp to importlib.util imp is deprecated, and has been removed in python 3.12+. This is largely following this example: https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly Bug: 388344853 Test: treehugger Change-Id: I3b4f7659377951d7c1291f11a977516924b25393 --- common.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/common.py b/common.py index c1fe47c..eb6320d 100755 --- a/common.py +++ b/common.py @@ -18,7 +18,7 @@ import errno import getopt import getpass -import imp +import importlib.util import os import platform import re @@ -1120,16 +1120,17 @@ def __init__(self, **kwargs): return try: if os.path.isdir(path): - info = imp.find_module("releasetools", [path]) - else: - d, f = os.path.split(path) - b, x = os.path.splitext(f) - if x == ".py": - f = b - info = imp.find_module(f, [d]) + path = os.path.join(path, "releasetools") + if os.path.isdir(path): + path = os.path.join(path, "__init__.py") + if not os.path.exists(path) and os.path.exists(path + ".py"): + path = path + ".py" + spec = importlib.util.spec_from_file_location("device_specific", path) print("loaded device-specific extensions from", path) - self.module = imp.load_module("device_specific", *info) - except ImportError: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self.module = module + except (ImportError, FileNotFoundError): print("unable to load device-specific module; assuming none") def _DoCall(self, function_name, *args, **kwargs): From d71aec88f8026d1d3c09dcf7cf6e93e7c67aabfa Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Fri, 17 Jan 2025 04:09:14 -0500 Subject: [PATCH 2/4] Fix releasetools device_specific error case When the specified device_specific code isn't found, handle the error with a log statement rather than a NoneType exception. Fixes: 390497895 Test: treehugger Change-Id: Id5925f4e9ec9ae35d217823fc32f1ac7f06f1652 --- common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common.py b/common.py index eb6320d..6cf5155 100755 --- a/common.py +++ b/common.py @@ -1126,6 +1126,8 @@ def __init__(self, **kwargs): if not os.path.exists(path) and os.path.exists(path + ".py"): path = path + ".py" spec = importlib.util.spec_from_file_location("device_specific", path) + if not spec: + raise FileNotFoundError(path) print("loaded device-specific extensions from", path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) From 3c22af3595bdb82e37781978c3a9b2fcf975281a Mon Sep 17 00:00:00 2001 From: Tianjie Xu Date: Thu, 6 Jul 2017 15:13:59 -0700 Subject: [PATCH 3/4] Generate brotli compressed {}.new.dat for full OTA Brotli has a better compression ratio than deflation. So for non-AB full OTA, we can compress the new.dat with brotli first and store it in the zip package. This increase the OTA generation time by ~5 minutes for a full OTA (measured based on 2.0G system.new.dat) Bug: 34220646 Test: Generate a full OTA for bullhead Change-Id: I9c0550af8eafcfa68711f74290c8e2d05a96648f --- common.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/common.py b/common.py index 6cf5155..ee1b66d 100755 --- a/common.py +++ b/common.py @@ -1533,9 +1533,34 @@ def _WriteUpdate(self, script, output_zip): ZipWrite(output_zip, '{}.transfer.list'.format(self.path), '{}.transfer.list'.format(self.partition)) - ZipWrite(output_zip, - '{}.new.dat'.format(self.path), - '{}.new.dat'.format(self.partition)) + + # For full OTA, compress the new.dat with brotli with quality 6 to reduce its size. Quailty 9 + # almost triples the compression time but doesn't further reduce the size too much. + # For a typical 1.8G system.new.dat + # zip | brotli(quality 6) | brotli(quality 9) + # compressed_size: 942M | 869M (~8% reduced) | 854M + # compression_time: 75s | 265s | 719s + # decompression_time: 15s | 25s | 25s + + if not self.src: + bro_cmd = ['bro', '--quality', '6', + '--input', '{}.new.dat'.format(self.path), + '--output', '{}.new.dat.br'.format(self.path)] + print("Compressing {}.new.dat with brotli".format(self.partition)) + p = Run(bro_cmd, stdout=subprocess.PIPE) + p.communicate() + assert p.returncode == 0,\ + 'compression of {}.new.dat failed'.format(self.partition) + + new_data_name = '{}.new.dat.br'.format(self.partition) + ZipWrite(output_zip, + '{}.new.dat.br'.format(self.path), + new_data_name, + compress_type=zipfile.ZIP_STORED) + else: + new_data_name = '{}.new.dat'.format(self.partition) + ZipWrite(output_zip, '{}.new.dat'.format(self.path), new_data_name) + ZipWrite(output_zip, '{}.patch.dat'.format(self.path), '{}.patch.dat'.format(self.partition), @@ -1548,9 +1573,10 @@ def _WriteUpdate(self, script, output_zip): call = ('block_image_update("{device}", ' 'package_extract_file("{partition}.transfer.list"), ' - '"{partition}.new.dat", "{partition}.patch.dat") ||\n' + '"{new_data_name}", "{partition}.patch.dat") ||\n' ' abort("E{code}: Failed to update {partition} image.");'.format( - device=self.device, partition=self.partition, code=code)) + device=self.device, partition=self.partition, + new_data_name=new_data_name, code=code)) script.AppendExtra(script.WordWrap(call)) def _HashBlocks(self, source, ranges): # pylint: disable=no-self-use From d412d84b15dd7cf2358afc49742eb4af64a8f1f5 Mon Sep 17 00:00:00 2001 From: Ricky Cheung Date: Tue, 3 Jun 2025 14:57:39 +0800 Subject: [PATCH 4/4] Add support for brotli compression Based on common.py implementation (commit 3c22af3). Signed-off-by: Ricky Cheung --- img2sdat.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/img2sdat.py b/img2sdat.py index 3ba2689..2ab2a88 100755 --- a/img2sdat.py +++ b/img2sdat.py @@ -9,9 +9,10 @@ from __future__ import print_function import sys, os, errno, tempfile +import brotli import common, blockimgdiff, sparse_img -def main(INPUT_IMAGE, OUTDIR='.', VERSION=None, PREFIX='system'): +def main(INPUT_IMAGE, OUTDIR='.', VERSION=None, PREFIX='system', BROTLI=False): global input __version__ = '1.7' @@ -65,6 +66,16 @@ def main(INPUT_IMAGE, OUTDIR='.', VERSION=None, PREFIX='system'): b = blockimgdiff.BlockImageDiff(image, None, VERSION) b.Compute(OUTDIR) + # Generate brotli compressed output files + if BROTLI: + print("Compressing {}.new.dat with brotli".format(PREFIX)) + with open('{}.new.dat'.format(PREFIX), 'rb') as infile: + data = infile.read() + outfile = open('{}.new.dat.br'.format(PREFIX), 'wb') + data = brotli.compress(data, mode=brotli.MODE_GENERIC, + quality=6, lgwin=22, lgblock=0) + outfile.write(data) + print('Done! Output files: %s' % os.path.dirname(OUTDIR)) return @@ -76,6 +87,7 @@ def main(INPUT_IMAGE, OUTDIR='.', VERSION=None, PREFIX='system'): parser.add_argument('-o', '--outdir', help='output directory (current directory by default)') parser.add_argument('-v', '--version', help='transfer list version number, will be asked by default - more info on xda thread)') parser.add_argument('-p', '--prefix', help='name of image (prefix.new.dat)') + parser.add_argument('--brotli', action='store_true', help='generate compressed brotli output') args = parser.parse_args() @@ -95,5 +107,7 @@ def main(INPUT_IMAGE, OUTDIR='.', VERSION=None, PREFIX='system'): PREFIX = args.prefix else: PREFIX = 'system' + + BROTLI = args.brotli - main(INPUT_IMAGE, OUTDIR, VERSION, PREFIX) + main(INPUT_IMAGE, OUTDIR, VERSION, PREFIX, BROTLI)