From 5b4bf5ef4ba77d3c91c3e54817b0e79fbd884357 Mon Sep 17 00:00:00 2001 From: Ansh Dadwal Date: Fri, 25 Jul 2025 23:44:39 +0530 Subject: [PATCH] `python`: add `3.14` support --- pythonforandroid/archs.py | 5 +- .../common/build/jni/application/src/start.c | 82 ++++++++--- .../java/org/kivy/android/PythonUtil.java | 39 +++-- pythonforandroid/recipe.py | 114 ++++++++++----- .../recipes/hostpython3/__init__.py | 42 ++++-- .../patches/pyconfig_detection.patch | 13 -- pythonforandroid/recipes/numpy/__init__.py | 5 +- pythonforandroid/recipes/openssl/__init__.py | 38 ++--- .../recipes/openssl/disable-sover.patch | 11 -- pythonforandroid/recipes/python3/__init__.py | 135 ++++++++++-------- .../python3/patches/3.14_armv7l_fix.patch | 12 ++ .../recipes/setuptools/__init__.py | 9 +- tests/recipes/test_openssl.py | 15 +- tests/recipes/test_python3.py | 11 +- tests/recipes/test_reportlab.py | 1 + 15 files changed, 302 insertions(+), 230 deletions(-) delete mode 100644 pythonforandroid/recipes/hostpython3/patches/pyconfig_detection.patch delete mode 100644 pythonforandroid/recipes/openssl/disable-sover.patch create mode 100644 pythonforandroid/recipes/python3/patches/3.14_armv7l_fix.patch diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index 4137768096..c2c1f91fc7 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -132,10 +132,7 @@ def get_env(self, with_flags_in_cc=True): env['CPPFLAGS'] = ' '.join(self.common_cppflags).format( ctx=self.ctx, command_prefix=self.command_prefix, - python_includes=join( - self.ctx.get_python_install_dir(self.arch), - 'include/python{}'.format(self.ctx.python_recipe.version[0:3]), - ), + python_includes=join(self.ctx.python_recipe.get_build_dir(self.arch), 'Include') ) # LDFLAGS: Link the extra global link paths first before anything else diff --git a/pythonforandroid/bootstraps/common/build/jni/application/src/start.c b/pythonforandroid/bootstraps/common/build/jni/application/src/start.c index ef910cab3d..1862208943 100644 --- a/pythonforandroid/bootstraps/common/build/jni/application/src/start.c +++ b/pythonforandroid/bootstraps/common/build/jni/application/src/start.c @@ -31,6 +31,7 @@ #define ENTRYPOINT_MAXLEN 128 #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) #define LOGP(x) LOG("python", (x)) +#define P4A_MIN_VER 11 static PyObject *androidembed_log(PyObject *self, PyObject *args) { char *logstr = NULL; @@ -154,11 +155,6 @@ int main(int argc, char *argv[]) { Py_NoSiteFlag=1; #endif -#if PY_MAJOR_VERSION < 3 - Py_SetProgramName("android_python"); -#else - Py_SetProgramName(L"android_python"); -#endif #if PY_MAJOR_VERSION >= 3 /* our logging module for android @@ -174,40 +170,80 @@ int main(int argc, char *argv[]) { char python_bundle_dir[256]; snprintf(python_bundle_dir, 256, "%s/_python_bundle", getenv("ANDROID_UNPACK")); - if (dir_exists(python_bundle_dir)) { - LOGP("_python_bundle dir exists"); - snprintf(paths, 256, - "%s/stdlib.zip:%s/modules", - python_bundle_dir, python_bundle_dir); - LOGP("calculated paths to be..."); - LOGP(paths); + #if PY_MAJOR_VERSION >= 3 - #if PY_MAJOR_VERSION >= 3 - wchar_t *wchar_paths = Py_DecodeLocale(paths, NULL); - Py_SetPath(wchar_paths); + #if PY_MINOR_VERSION >= P4A_MIN_VER + PyConfig config; + PyConfig_InitPythonConfig(&config); + config.program_name = L"android_python"; + #else + Py_SetProgramName(L"android_python"); #endif - LOGP("set wchar paths..."); + #else + Py_SetProgramName("android_python"); + #endif + + if (dir_exists(python_bundle_dir)) { + LOGP("_python_bundle dir exists"); + + #if PY_MAJOR_VERSION >= 3 + #if PY_MINOR_VERSION >= P4A_MIN_VER + + wchar_t wchar_zip_path[256]; + wchar_t wchar_modules_path[256]; + swprintf(wchar_zip_path, 256, L"%s/stdlib.zip", python_bundle_dir); + swprintf(wchar_modules_path, 256, L"%s/modules", python_bundle_dir); + + config.module_search_paths_set = 1; + PyWideStringList_Append(&config.module_search_paths, wchar_zip_path); + PyWideStringList_Append(&config.module_search_paths, wchar_modules_path); + #else + char paths[512]; + snprintf(paths, 512, "%s/stdlib.zip:%s/modules", python_bundle_dir, python_bundle_dir); + wchar_t *wchar_paths = Py_DecodeLocale(paths, NULL); + Py_SetPath(wchar_paths); + #endif + + #endif + + LOGP("set wchar paths..."); } else { LOGP("_python_bundle does not exist...this not looks good, all python" " recipes should have this folder, should we expect a crash soon?"); } - Py_Initialize(); +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= P4A_MIN_VER + PyStatus status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + LOGP("Python initialization failed:"); + LOGP(status.err_msg); + } +#else + Py_Initialize(); + LOGP("Python initialized using legacy Py_Initialize()."); +#endif + LOGP("Initialized python"); - /* ensure threads will work. - */ - LOGP("AND: Init threads"); - PyEval_InitThreads(); + /* < 3.9 requires explicit GIL initialization + * 3.9+ PyEval_InitThreads() is deprecated and unnecessary + */ + #if PY_VERSION_HEX < 0x03090000 + LOGP("Initializing threads (required for Python < 3.9)"); + PyEval_InitThreads(); + #endif #if PY_MAJOR_VERSION < 3 initandroidembed(); #endif - PyRun_SimpleString("import androidembed\nandroidembed.log('testing python " - "print redirection')"); + PyRun_SimpleString( + "import androidembed\n" + "androidembed.log('testing python print redirection')" + + ); /* inject our bootstrap code to redirect python stdin/stdout * replace sys.path with our path diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java index 83d11639bb..065f43c3bd 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java @@ -39,27 +39,22 @@ protected static void addLibraryIfExists(ArrayList libsList, String patt } protected static ArrayList getLibraries(File libsDir) { - ArrayList libsList = new ArrayList(); - addLibraryIfExists(libsList, "sqlite3", libsDir); - addLibraryIfExists(libsList, "ffi", libsDir); - addLibraryIfExists(libsList, "png16", libsDir); - addLibraryIfExists(libsList, "ssl.*", libsDir); - addLibraryIfExists(libsList, "crypto.*", libsDir); - addLibraryIfExists(libsList, "SDL2", libsDir); - addLibraryIfExists(libsList, "SDL2_image", libsDir); - addLibraryIfExists(libsList, "SDL2_mixer", libsDir); - addLibraryIfExists(libsList, "SDL2_ttf", libsDir); - addLibraryIfExists(libsList, "SDL3", libsDir); - addLibraryIfExists(libsList, "SDL3_image", libsDir); - addLibraryIfExists(libsList, "SDL3_mixer", libsDir); - addLibraryIfExists(libsList, "SDL3_ttf", libsDir); - libsList.add("python3.5m"); - libsList.add("python3.6m"); - libsList.add("python3.7m"); - libsList.add("python3.8"); - libsList.add("python3.9"); - libsList.add("python3.10"); - libsList.add("python3.11"); + ArrayList libsList = new ArrayList<>(); + + String[] libNames = { + "sqlite3", "ffi", "png16", "ssl.*", "crypto.*", + "SDL2", "SDL2_image", "SDL2_mixer", "SDL2_ttf", + "SDL3", "SDL3_image", "SDL3_mixer", "SDL3_ttf" + }; + + for (String name : libNames) { + addLibraryIfExists(libsList, name, libsDir); + } + + for (int v = 5; v <= 14; v++) { + libsList.add("python3." + v + (v <= 7 ? "m" : "")); + } + libsList.add("main"); return libsList; } @@ -79,7 +74,7 @@ public static void loadLibraries(File filesDir, File libsDir) { // load, and it has failed, give a more // general error Log.v(TAG, "Library loading error: " + e.getMessage()); - if (lib.startsWith("python3.11") && !foundPython) { + if (lib.startsWith("python3.14") && !foundPython) { throw new RuntimeException("Could not load any libpythonXXX.so"); } else if (lib.startsWith("python")) { continue; diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 44aab013a6..8808ffc0d2 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -12,6 +12,7 @@ from urllib.request import urlretrieve from os import listdir, unlink, environ, curdir, walk from sys import stdout +from multiprocessing import cpu_count import time try: from urlparse import urlparse @@ -517,7 +518,7 @@ def unpack(self, arch): for entry in listdir(extraction_filename): # Previously we filtered out the .git folder, but during the build process for some recipes # (e.g. when version is parsed by `setuptools_scm`) that may be needed. - shprint(sh.cp, '-Rv', + shprint(sh.cp, '-R', join(extraction_filename, entry), directory_name) else: @@ -830,6 +831,8 @@ def build_arch(self, arch, *extra_args): shprint( sh.Command(join(self.ctx.ndk_dir, "ndk-build")), 'V=1', + "-j", + str(cpu_count()), 'NDK_DEBUG=' + ("1" if self.ctx.build_as_debuggable else "0"), 'APP_PLATFORM=android-' + str(self.ctx.ndk_api), 'APP_ABI=' + arch.arch, @@ -878,6 +881,8 @@ class PythonRecipe(Recipe): hostpython_prerequisites = [] '''List of hostpython packages required to build a recipe''' + _host_recipe = None + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if 'python3' not in self.depends: @@ -890,6 +895,10 @@ def __init__(self, *args, **kwargs): depends = list(set(depends)) self.depends = depends + def prebuild_arch(self, arch): + self._host_recipe = Recipe.get_recipe("hostpython3", self.ctx) + return super().prebuild_arch(arch) + def clean_build(self, arch=None): super().clean_build(arch=arch) name = self.folder_name @@ -907,8 +916,7 @@ def clean_build(self, arch=None): def real_hostpython_location(self): host_name = 'host{}'.format(self.ctx.python_recipe.name) if host_name == 'hostpython3': - python_recipe = Recipe.get_recipe(host_name, self.ctx) - return python_recipe.python_exe + return self._host_recipe.python_exe else: python_recipe = self.ctx.python_recipe return 'python{}'.format(python_recipe.version) @@ -927,14 +935,50 @@ def folder_name(self): name = self.name return name + def patch_shebang(self, _file, original_bin): + _file_des = open(_file, "r") + + try: + data = _file_des.readlines() + except UnicodeDecodeError: + return + + if "#!" in (line := data[0]): + if line.split("#!")[-1].strip() == original_bin: + return + + info(f"Fixing shebang for '{_file}'") + data.pop(0) + data.insert(0, "#!" + original_bin + "\n") + _file_des.close() + _file_des = open(_file, "w") + _file_des.write("".join(data)) + _file_des.close() + + def patch_shebangs(self, path, original_bin): + if not isdir(path): + warning(f"Shebang patch skipped: '{path}' does not exist.") + return + # set correct shebang + for file in listdir(path): + _file = join(path, file) + self.patch_shebang(_file, original_bin) + def get_recipe_env(self, arch=None, with_flags_in_cc=True): + if self._host_recipe is None: + self._host_recipe = Recipe.get_recipe("hostpython3", self.ctx) + env = super().get_recipe_env(arch, with_flags_in_cc) - env['PYTHONNOUSERSITE'] = '1' # Set the LANG, this isn't usually important but is a better default # as it occasionally matters how Python e.g. reads files env['LANG'] = "en_GB.UTF-8" + # Binaries made by packages installed by pip - env["PATH"] = join(self.hostpython_site_dir, "bin") + ":" + env["PATH"] + self.patch_shebangs(self._host_recipe.local_bin, self.real_hostpython_location) + env["PATH"] = self._host_recipe.local_bin + ":" + self._host_recipe.site_bin + ":" + env["PATH"] + + host_env = self.get_hostrecipe_env() + env['PYTHONPATH'] = host_env["PYTHONPATH"] if not self.call_hostpython_via_targetpython: env['CFLAGS'] += ' -I{}'.format( @@ -945,18 +989,6 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): self.ctx.python_recipe.link_version, ) - hppath = [] - hppath.append(join(dirname(self.hostpython_location), 'Lib')) - hppath.append(join(hppath[0], 'site-packages')) - builddir = join(dirname(self.hostpython_location), 'build') - if exists(builddir): - hppath += [join(builddir, d) for d in listdir(builddir) - if isdir(join(builddir, d))] - if len(hppath) > 0: - if 'PYTHONPATH' in env: - env['PYTHONPATH'] = ':'.join(hppath + [env['PYTHONPATH']]) - else: - env['PYTHONPATH'] = ':'.join(hppath) return env def should_build(self, arch): @@ -988,18 +1020,25 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True): hostpython = sh.Command(self.hostpython_location) hpenv = env.copy() with current_directory(self.get_build_dir(arch.arch)): - shprint(hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), - '--install-lib=.', - _env=hpenv, *self.setup_extra_args) - # If asked, also install in the hostpython build dir - if self.install_in_hostpython: - self.install_hostpython_package(arch) + if isfile("setup.py"): + shprint(hostpython, 'setup.py', 'install', '-O2', + '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), + '--install-lib=.', + _env=hpenv, *self.setup_extra_args) - def get_hostrecipe_env(self, arch): + # If asked, also install in the hostpython build dir + if self.install_in_hostpython: + self.install_hostpython_package(arch) + else: + warning("`PythonRecipe.install_python_package` called without `setup.py` file!") + + def get_hostrecipe_env(self, arch=None): env = environ.copy() - env['PYTHONPATH'] = self.hostpython_site_dir + _python_path = self._host_recipe.get_path_to_python() + libdir = glob.glob(join(_python_path, "build", "lib*")) + env['PYTHONPATH'] = self._host_recipe.site_dir + ":" + join( + _python_path, "Modules") + ":" + (libdir[0] if libdir else "") return env @property @@ -1010,8 +1049,8 @@ def install_hostpython_package(self, arch): env = self.get_hostrecipe_env(arch) real_hostpython = sh.Command(self.real_hostpython_location) shprint(real_hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(dirname(self.real_hostpython_location)), '--install-lib=Lib/site-packages', + '--root={}'.format(self._host_recipe.site_root), _env=env, *self.setup_extra_args) @property @@ -1029,7 +1068,7 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True): pip_options = [ "install", *packages, - "--target", self.hostpython_site_dir, "--python-version", + "--target", self._host_recipe.site_dir, "--python-version", self.ctx.python_recipe.version, # Don't use sources, instead wheels "--only-binary=:all:", @@ -1037,7 +1076,9 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True): if force_upgrade: pip_options.append("--upgrade") # Use system's pip - shprint(sh.pip, *pip_options) + pip_env = self.get_hostrecipe_env() + pip_env["HOME"] = "/tmp" + shprint(sh.Command(self.real_hostpython_location), "-m", "pip", *pip_options, _env=pip_env) def restore_hostpython_prerequisites(self, packages): _packages = [] @@ -1231,7 +1272,7 @@ def get_recipe_env(self, arch, **kwargs): return env def get_wheel_platform_tag(self, arch): - return "android_" + { + return f"android_{self.ctx.ndk_api}_" + { "armeabi-v7a": "arm", "arm64-v8a": "aarch64", "x86_64": "x86_64", @@ -1265,10 +1306,17 @@ def install_wheel(self, arch, built_wheels): wf.close() def build_arch(self, arch): + + build_dir = self.get_build_dir(arch.arch) + if not (isfile(join(build_dir, "pyproject.toml")) or isfile(join(build_dir, "setup.py"))): + warning("Skipping build because it does not appear to be a Python project.") + return + self.install_hostpython_prerequisites( - packages=["build[virtualenv]", "pip"] + self.hostpython_prerequisites + packages=["build[virtualenv]", "pip", "setuptools"] + self.hostpython_prerequisites ) - build_dir = self.get_build_dir(arch.arch) + self.patch_shebangs(self._host_recipe.site_bin, self.real_hostpython_location) + env = self.get_recipe_env(arch, with_flags_in_cc=True) # make build dir separately sub_build_dir = join(build_dir, "p4a_android_build") @@ -1423,7 +1471,7 @@ def get_recipe_env(self, arch, **kwargs): env["PYO3_CROSS_LIB_DIR"] = realpath(glob.glob(join( realpython_dir, "android-build", "build", - "lib.linux-*-{}/".format(self.python_major_minor_version), + "lib.*{}/".format(self.python_major_minor_version), ))[0]) info_main("Ensuring rust build toolchain") diff --git a/pythonforandroid/recipes/hostpython3/__init__.py b/pythonforandroid/recipes/hostpython3/__init__.py index 9ba4580019..094660fada 100644 --- a/pythonforandroid/recipes/hostpython3/__init__.py +++ b/pythonforandroid/recipes/hostpython3/__init__.py @@ -5,6 +5,7 @@ from pathlib import Path from os.path import join +from packaging.version import Version from pythonforandroid.logger import shprint from pythonforandroid.recipe import Recipe from pythonforandroid.util import ( @@ -35,18 +36,15 @@ class HostPython3Recipe(Recipe): :class:`~pythonforandroid.python.HostPythonRecipe` ''' - version = '3.11.5' - name = 'hostpython3' + version = '3.11.13' - build_subdir = 'native-build' - '''Specify the sub build directory for the hostpython3 recipe. Defaults - to ``native-build``.''' - - url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' + url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz' '''The default url to download our host python recipe. This url will change depending on the python version set in attribute :attr:`version`.''' - patches = ['patches/pyconfig_detection.patch'] + build_subdir = 'native-build' + '''Specify the sub build directory for the hostpython3 recipe. Defaults + to ``native-build``.''' @property def _exe_name(self): @@ -95,6 +93,26 @@ def get_build_dir(self, arch=None): def get_path_to_python(self): return join(self.get_build_dir(), self.build_subdir) + @property + def site_root(self): + return join(self.get_path_to_python(), "root") + + @property + def site_bin(self): + return join(self.site_root, self.site_dir, "bin") + + @property + def local_bin(self): + return join(self.site_root, "usr/local/bin/") + + @property + def site_dir(self): + p_version = Version(self.version) + return join( + self.site_root, + f"usr/local/lib/python{p_version.major}.{p_version.minor}/site-packages/" + ) + def build_arch(self, arch): env = self.get_recipe_env(arch) @@ -105,9 +123,11 @@ def build_arch(self, arch): ensure_dir(build_dir) # Configure the build + build_configured = False with current_directory(build_dir): if not Path('config.status').exists(): shprint(sh.Command(join(recipe_build_dir, 'configure')), _env=env) + build_configured = True with current_directory(recipe_build_dir): # Create the Setup file. This copying from Setup.dist is @@ -138,7 +158,13 @@ def build_arch(self, arch): shprint(sh.cp, exe, self.python_exe) break + ensure_dir(self.site_root) self.ctx.hostpython = self.python_exe + if build_configured: + shprint( + sh.Command(self.python_exe), "-m", "ensurepip", "--root", self.site_root, "-U", + _env={"HOME": "/tmp"} + ) recipe = HostPython3Recipe() diff --git a/pythonforandroid/recipes/hostpython3/patches/pyconfig_detection.patch b/pythonforandroid/recipes/hostpython3/patches/pyconfig_detection.patch deleted file mode 100644 index 7f78b664e1..0000000000 --- a/pythonforandroid/recipes/hostpython3/patches/pyconfig_detection.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff -Nru Python-3.8.2/Lib/site.py Python-3.8.2-new/Lib/site.py ---- Python-3.8.2/Lib/site.py 2020-04-28 12:48:38.000000000 -0700 -+++ Python-3.8.2-new/Lib/site.py 2020-04-28 12:52:46.000000000 -0700 -@@ -487,7 +487,8 @@ - if key == 'include-system-site-packages': - system_site = value.lower() - elif key == 'home': -- sys._home = value -+ # this is breaking pyconfig.h path detection with venv -+ print('Ignoring "sys._home = value" override', file=sys.stderr) - - sys.prefix = sys.exec_prefix = site_prefix - diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index f85e44d4f5..140ff849d8 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -40,10 +40,9 @@ def build_arch(self, arch): super().build_arch(arch) self.restore_hostpython_prerequisites(["cython"]) - def get_hostrecipe_env(self, arch): - env = super().get_hostrecipe_env(arch) + def get_hostrecipe_env(self, arch=None): + env = super().get_hostrecipe_env(arch=arch) env['RANLIB'] = shutil.which('ranlib') - env["LDFLAGS"] += " -lm" return env diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index 766c10e361..9a9a8c8a0f 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -1,4 +1,5 @@ from os.path import join +from multiprocessing import cpu_count from pythonforandroid.recipe import Recipe from pythonforandroid.util import current_directory @@ -44,35 +45,23 @@ class OpenSSLRecipe(Recipe): ''' - version = '1.1' - '''the major minor version used to link our recipes''' - - url_version = '1.1.1w' - '''the version used to download our libraries''' - - url = 'https://www.openssl.org/source/openssl-{url_version}.tar.gz' + version = '3.3.1' + url = 'https://www.openssl.org/source/openssl-{version}.tar.gz' built_libraries = { - 'libcrypto{version}.so'.format(version=version): '.', - 'libssl{version}.so'.format(version=version): '.', + 'libcrypto.so': '.', + 'libssl.so': '.', } - @property - def versioned_url(self): - if self.url is None: - return None - return self.url.format(url_version=self.url_version) - def get_build_dir(self, arch): return join( - self.get_build_container_dir(arch), self.name + self.version + self.get_build_container_dir(arch), self.name + self.version[0] ) def include_flags(self, arch): '''Returns a string with the include folders''' openssl_includes = join(self.get_build_dir(arch.arch), 'include') return (' -I' + openssl_includes + - ' -I' + join(openssl_includes, 'internal') + ' -I' + join(openssl_includes, 'openssl')) def link_dirs_flags(self, arch): @@ -85,7 +74,7 @@ def link_libs_flags(self): '''Returns a string with the appropriate `-l` flags to link with the openssl libs. This string is usually added to the environment variable `LIBS`''' - return ' -lcrypto{version} -lssl{version}'.format(version=self.version) + return ' -lcrypto -lssl' def link_flags(self, arch): '''Returns a string with the flags to link with the openssl libraries @@ -94,10 +83,12 @@ def link_flags(self, arch): def get_recipe_env(self, arch=None): env = super().get_recipe_env(arch) - env['OPENSSL_VERSION'] = self.version - env['MAKE'] = 'make' # This removes the '-j5', which isn't safe + env['OPENSSL_VERSION'] = self.version[0] env['CC'] = 'clang' - env['ANDROID_NDK_HOME'] = self.ctx.ndk_dir + env['ANDROID_NDK_ROOT'] = self.ctx.ndk_dir + env["PATH"] = f"{self.ctx.ndk.llvm_bin_dir}:{env['PATH']}" + env["CFLAGS"] += " -Wno-macro-redefined" + env["MAKE"] = "make" return env def select_build_arch(self, arch): @@ -125,13 +116,12 @@ def build_arch(self, arch): 'shared', 'no-dso', 'no-asm', + 'no-tests', buildarch, '-D__ANDROID_API__={}'.format(self.ctx.ndk_api), ] shprint(perl, 'Configure', *config_args, _env=env) - self.apply_patch('disable-sover.patch', arch.arch) - - shprint(sh.make, 'build_libs', _env=env) + shprint(sh.make, '-j', str(cpu_count()), _env=env) recipe = OpenSSLRecipe() diff --git a/pythonforandroid/recipes/openssl/disable-sover.patch b/pythonforandroid/recipes/openssl/disable-sover.patch deleted file mode 100644 index d944483cda..0000000000 --- a/pythonforandroid/recipes/openssl/disable-sover.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- openssl/Makefile.orig 2018-10-20 22:49:40.418310423 +0200 -+++ openssl/Makefile 2018-10-20 22:50:23.347322403 +0200 -@@ -19,7 +19,7 @@ - SHLIB_MAJOR=1 - SHLIB_MINOR=1 - SHLIB_TARGET=linux-shared --SHLIB_EXT=.so.$(SHLIB_VERSION_NUMBER) -+SHLIB_EXT=$(SHLIB_VERSION_NUMBER).so - SHLIB_EXT_SIMPLE=.so - SHLIB_EXT_IMPORT= - diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index 7cd3928d76..81aee7c66e 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -3,12 +3,11 @@ import subprocess from os import environ, utime -from os.path import dirname, exists, join -from pathlib import Path +from os.path import dirname, exists, join, isfile import shutil -from pythonforandroid.logger import info, warning, shprint -from pythonforandroid.patching import version_starts_with +from packaging.version import Version +from pythonforandroid.logger import info, shprint, warning from pythonforandroid.recipe import Recipe, TargetPythonRecipe from pythonforandroid.util import ( current_directory, @@ -55,34 +54,37 @@ class Python3Recipe(TargetPythonRecipe): :class:`~pythonforandroid.python.GuestPythonRecipe` ''' - version = '3.11.5' - url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' + version = '3.11.13' + _p_version = Version(version) + url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz' name = 'python3' patches = [ 'patches/pyconfig_detection.patch', 'patches/reproducible-buildinfo.diff', - - # Python 3.7.1 - ('patches/py3.7.1_fix-ctypes-util-find-library.patch', version_starts_with("3.7")), - ('patches/py3.7.1_fix-zlib-version.patch', version_starts_with("3.7")), - - # Python 3.8.1 & 3.9.X - ('patches/py3.8.1.patch', version_starts_with("3.8")), - ('patches/py3.8.1.patch', version_starts_with("3.9")), - ('patches/py3.8.1.patch', version_starts_with("3.10")), - ('patches/cpython-311-ctypes-find-library.patch', version_starts_with("3.11")), ] - if shutil.which('lld') is not None: + if _p_version.major == 3 and _p_version.minor == 7: patches += [ - ("patches/py3.7.1_fix_cortex_a8.patch", version_starts_with("3.7")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.8")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.9")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.11")), + 'patches/py3.7.1_fix-ctypes-util-find-library.patch', + 'patches/py3.7.1_fix-zlib-version.patch', ] + if 8 <= _p_version.minor <= 10: + patches.append('patches/py3.8.1.patch') + + if _p_version.minor >= 11: + patches.append('patches/cpython-311-ctypes-find-library.patch') + + if _p_version.minor >= 14: + patches.append('patches/3.14_armv7l_fix.patch') + + if shutil.which('lld') is not None: + if _p_version.minor == 7: + patches.append("patches/py3.7.1_fix_cortex_a8.patch") + elif _p_version.minor >= 8: + patches.append("patches/py3.8.1_fix_cortex_a8.patch") + depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi'] # those optional depends allow us to build python compression modules: # - _bz2.so @@ -90,23 +92,33 @@ class Python3Recipe(TargetPythonRecipe): opt_depends = ['libbz2', 'liblzma'] '''The optional libraries which we would like to get our python linked''' - configure_args = ( + configure_args = [ '--host={android_host}', '--build={android_build}', '--enable-shared', '--enable-ipv6', - 'ac_cv_file__dev_ptmx=yes', - 'ac_cv_file__dev_ptc=no', + '--enable-loadable-sqlite-extensions', + '--without-static-libpython', + '--without-readline', '--without-ensurepip', - 'ac_cv_little_endian_double=yes', - 'ac_cv_header_sys_eventfd_h=no', + + # Android prefix '--prefix={prefix}', '--exec-prefix={exec_prefix}', - '--enable-loadable-sqlite-extensions' - ) + '--enable-loadable-sqlite-extensions', + + # Special cross compile args + 'ac_cv_file__dev_ptmx=yes', + 'ac_cv_file__dev_ptc=no', + 'ac_cv_header_sys_eventfd_h=no', + 'ac_cv_little_endian_double=yes', + 'ac_cv_header_bzlib_h=no', + ] - if version_starts_with("3.11"): - configure_args += ('--with-build-python={python_host_bin}',) + if _p_version.minor >= 11: + configure_args.extend([ + '--with-build-python={python_host_bin}', + ]) '''The configure arguments needed to build the python recipe. Those are used in method :meth:`build_arch` (if not overwritten like python3's @@ -168,6 +180,9 @@ class Python3Recipe(TargetPythonRecipe): longer used and has been removed in favour of extension .pyc ''' + disable_gil = False + '''python3.13 experimental free-threading build''' + def __init__(self, *args, **kwargs): self._ctx = None super().__init__(*args, **kwargs) @@ -199,7 +214,7 @@ def link_root(self, arch_name): return join(self.get_build_dir(arch_name), 'android-build') def should_build(self, arch): - return not Path(self.link_root(arch.arch), self._libpython).is_file() + return not isfile(join(self.link_root(arch.arch), self._libpython)) def prebuild_arch(self, arch): super().prebuild_arch(arch) @@ -244,30 +259,26 @@ def add_flags(include_flags, link_dirs, link_libs): env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs env['LIBS'] = env.get('LIBS', '') + link_libs - if 'sqlite3' in self.ctx.recipe_build_order: - info('Activating flags for sqlite3') - recipe = Recipe.get_recipe('sqlite3', self.ctx) - add_flags(' -I' + recipe.get_build_dir(arch.arch), - ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3') - - if 'libffi' in self.ctx.recipe_build_order: - info('Activating flags for libffi') - recipe = Recipe.get_recipe('libffi', self.ctx) - # In order to force the correct linkage for our libffi library, we - # set the following variable to point where is our libffi.pc file, - # because the python build system uses pkg-config to configure it. - env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch) - add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)), - ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'), - ' -lffi') - - if 'openssl' in self.ctx.recipe_build_order: - info('Activating flags for openssl') - recipe = Recipe.get_recipe('openssl', self.ctx) - self.configure_args += \ - ('--with-openssl=' + recipe.get_build_dir(arch.arch),) - add_flags(recipe.include_flags(arch), - recipe.link_dirs_flags(arch), recipe.link_libs_flags()) + info('Activating flags for sqlite3') + recipe = Recipe.get_recipe('sqlite3', self.ctx) + add_flags(' -I' + recipe.get_build_dir(arch.arch), + ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3') + + info('Activating flags for libffi') + recipe = Recipe.get_recipe('libffi', self.ctx) + # In order to force the correct linkage for our libffi library, we + # set the following variable to point where is our libffi.pc file, + # because the python build system uses pkg-config to configure it. + env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch) + add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)), + ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'), + ' -lffi') + + info('Activating flags for openssl') + recipe = Recipe.get_recipe('openssl', self.ctx) + self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch)) + add_flags(recipe.include_flags(arch), + recipe.link_dirs_flags(arch), recipe.link_libs_flags()) for library_name in {'libbz2', 'liblzma'}: if library_name in self.ctx.recipe_build_order: @@ -303,6 +314,9 @@ def add_flags(include_flags, link_dirs, link_libs): env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '') add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz') + if self._p_version.minor >= 13 and self.disable_gil: + self.configure_args.append("--disable-gil") + return env def build_arch(self, arch): @@ -344,9 +358,6 @@ def build_arch(self, arch): exec_prefix=sys_exec_prefix)).split(' '), _env=env) - # Python build does not seem to play well with make -j option from Python 3.11 and onwards - # Before losing some time, please check issue - # https://github.com/python/cpython/issues/101295 , as the root cause looks similar shprint( sh.make, 'all', @@ -382,11 +393,13 @@ def create_python_bundle(self, dirn, arch): self.get_build_dir(arch.arch), 'android-build', 'build', - 'lib.linux{}-{}-{}'.format( + 'lib.{}{}-{}-{}'.format( + # android is now supported platform + "android" if self._p_version.minor >= 13 else "linux", '2' if self.version[0] == '2' else '', arch.command_prefix.split('-')[0], self.major_minor_version_string - )) + )) # Compile to *.pyc the python modules self.compile_python_files(modules_build_dir) diff --git a/pythonforandroid/recipes/python3/patches/3.14_armv7l_fix.patch b/pythonforandroid/recipes/python3/patches/3.14_armv7l_fix.patch new file mode 100644 index 0000000000..7565489a28 --- /dev/null +++ b/pythonforandroid/recipes/python3/patches/3.14_armv7l_fix.patch @@ -0,0 +1,12 @@ +diff '--color=auto' -uNr cpython-3.14.0/Lib/sysconfig/__init__.py cpython-3.14.0.mod/Lib/sysconfig/__init__.py +--- cpython-3.14.0/Lib/sysconfig/__init__.py 2025-10-07 21:45:41.236149298 +0530 ++++ cpython-3.14.0.mod/Lib/sysconfig/__init__.py 2025-10-07 21:45:54.650245131 +0530 +@@ -702,7 +702,7 @@ + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", +- "armv7l": "armeabi_v7a", ++ "arm": "armeabi_v7a", + }[machine] + elif osname == "linux": + # At least on Linux/Intel, 'machine' is the processor -- diff --git a/pythonforandroid/recipes/setuptools/__init__.py b/pythonforandroid/recipes/setuptools/__init__.py index 02b205023d..0c77b9fde1 100644 --- a/pythonforandroid/recipes/setuptools/__init__.py +++ b/pythonforandroid/recipes/setuptools/__init__.py @@ -1,11 +1,8 @@ -from pythonforandroid.recipe import PythonRecipe +from pythonforandroid.recipe import PyProjectRecipe -class SetuptoolsRecipe(PythonRecipe): - version = '69.2.0' - url = 'https://pypi.python.org/packages/source/s/setuptools/setuptools-{version}.tar.gz' - call_hostpython_via_targetpython = False - install_in_hostpython = True +class SetuptoolsRecipe(PyProjectRecipe): + hostpython_prerequisites = ['setuptools'] recipe = SetuptoolsRecipe() diff --git a/tests/recipes/test_openssl.py b/tests/recipes/test_openssl.py index f7ed362f68..e0910f59f3 100644 --- a/tests/recipes/test_openssl.py +++ b/tests/recipes/test_openssl.py @@ -11,7 +11,6 @@ class TestOpensslRecipe(BaseTestForMakeRecipe, unittest.TestCase): recipe_name = "openssl" sh_command_calls = ["perl"] - @mock.patch("pythonforandroid.recipes.openssl.sh.patch") @mock.patch("pythonforandroid.util.chdir") @mock.patch("pythonforandroid.build.ensure_dir") @mock.patch("shutil.which") @@ -20,31 +19,21 @@ def test_build_arch( mock_shutil_which, mock_ensure_dir, mock_current_directory, - mock_sh_patch, ): # We overwrite the base test method because we need to mock a little # more with this recipe. super().test_build_arch() - # make sure that the mocked methods are actually called - mock_sh_patch.assert_called() - - def test_versioned_url(self): - self.assertEqual( - self.recipe.url.format(url_version=self.recipe.url_version), - self.recipe.versioned_url, - ) def test_include_flags(self): inc = self.recipe.include_flags(self.arch) build_dir = self.recipe.get_build_dir(self.arch) - for i in {"include/internal", "include/openssl"}: + for i in {"include", "include/openssl"}: self.assertIn(f"-I{build_dir}/{i}", inc) def test_link_flags(self): build_dir = self.recipe.get_build_dir(self.arch) - openssl_version = self.recipe.version self.assertEqual( - f" -L{build_dir} -lcrypto{openssl_version} -lssl{openssl_version}", + f" -L{build_dir} -lcrypto -lssl", self.recipe.link_flags(self.arch), ) diff --git a/tests/recipes/test_python3.py b/tests/recipes/test_python3.py index f1d652b6c1..01d58f7d27 100644 --- a/tests/recipes/test_python3.py +++ b/tests/recipes/test_python3.py @@ -26,14 +26,6 @@ def test_property__libpython(self): f'libpython{self.recipe.link_version}.so' ) - @mock.patch('pythonforandroid.recipes.python3.Path.is_file') - def test_should_build(self, mock_is_file): - # in case that python lib exists, we shouldn't trigger the build - self.assertFalse(self.recipe.should_build(self.arch)) - # in case that python lib doesn't exist, we should trigger the build - mock_is_file.return_value = False - self.assertTrue(self.recipe.should_build(self.arch)) - def test_include_root(self): expected_include_dir = join( self.recipe.get_build_dir(self.arch.arch), 'Include', @@ -186,7 +178,8 @@ def test_create_python_bundle( recipe_build_dir, 'android-build', 'build', - 'lib.linux{}-{}-{}'.format( + 'lib.{}{}-{}-{}'.format( + 'android' if self.recipe.version[2] >= "3" else 'linux', '2' if self.recipe.version[0] == '2' else '', self.arch.command_prefix.split('-')[0], self.recipe.major_minor_version_string diff --git a/tests/recipes/test_reportlab.py b/tests/recipes/test_reportlab.py index 6129a6a963..cde17fd532 100644 --- a/tests/recipes/test_reportlab.py +++ b/tests/recipes/test_reportlab.py @@ -32,6 +32,7 @@ def test_prebuild_arch(self): patch('sh.patch'), \ patch('pythonforandroid.recipe.touch'), \ patch('sh.unzip'), \ + patch('pythonforandroid.recipe.Recipe.is_patched', lambda *a: False), \ patch('os.path.isfile'): self.recipe.prebuild_arch(self.arch) # makes sure placeholder got replaced with library and include paths