diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 8b1c723423..243e46f001 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -2,9 +2,11 @@ import copy import glob import os +import json +import tempfile from os import environ from os.path import ( - abspath, join, realpath, dirname, expanduser, exists + abspath, join, realpath, dirname, expanduser, exists, basename ) import re import shutil @@ -12,9 +14,12 @@ import sh +from packaging.utils import parse_wheel_filename +from packaging.requirements import Requirement + from pythonforandroid.androidndk import AndroidNDK from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64 -from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint) +from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint, Out_Style, Out_Fore) from pythonforandroid.pythonpackage import get_package_name from pythonforandroid.recipe import CythonRecipe, Recipe from pythonforandroid.recommendations import ( @@ -90,6 +95,8 @@ class Context: recipe_build_order = None # Will hold the list of all built recipes + python_modules = None # Will hold resolved pure python packages + symlink_bootstrap_files = False # If True, will symlink instead of copying during build java_build_tool = 'auto' @@ -444,6 +451,12 @@ def has_package(self, name, arch=None): # Failed to look up any meaningful name. return False + # normalize name to remove version tags + try: + name = Requirement(name).name + except Exception: + pass + # Try to look up recipe by name: try: recipe = Recipe.get_recipe(name, self) @@ -649,6 +662,104 @@ def run_setuppy_install(ctx, project_dir, env=None, arch=None): os.remove("._tmp_p4a_recipe_constraints.txt") +def is_wheel_platform_independent(whl_name): + name, version, build, tags = parse_wheel_filename(whl_name) + return all(tag.platform == "any" for tag in tags) + + +def process_python_modules(ctx, modules): + """Use pip --dry-run to resolve dependencies and filter for pure-Python packages + """ + modules = list(modules) + build_order = list(ctx.recipe_build_order) + + _requirement_names = [] + processed_modules = [] + + for module in modules+build_order: + try: + # we need to normalize names + # eg Requests>=2.0 becomes requests + _requirement_names.append(Requirement(module).name) + except Exception: + # name parsing failed; skip processing this module via pip + processed_modules.append(module) + if module in modules: + modules.remove(module) + + if len(processed_modules) > 0: + warning(f'Ignored by module resolver : {processed_modules}') + + # preserve the original module list + processed_modules.extend(modules) + + # temp file for pip report + fd, path = tempfile.mkstemp() + os.close(fd) + + # setup hostpython recipe + host_recipe = Recipe.get_recipe("hostpython3", ctx) + + env = environ.copy() + _python_path = host_recipe.get_path_to_python() + libdir = glob.glob(join(_python_path, "build", "lib*")) + env['PYTHONPATH'] = host_recipe.site_dir + ":" + join( + _python_path, "Modules") + ":" + (libdir[0] if libdir else "") + + shprint( + host_recipe.pip, 'install', *modules, + '--dry-run', '--break-system-packages', '--ignore-installed', + '--report', path, '-q', _env=env + ) + + with open(path, "r") as f: + report = json.load(f) + + os.remove(path) + + info('Extra resolved pure python dependencies :') + + ignored_str = " (ignored)" + # did we find any non pure python package? + any_not_pure_python = False + + # just for style + info(" ") + for module in report["install"]: + + mname = module["metadata"]["name"] + mver = module["metadata"]["version"] + filename = basename(module["download_info"]["url"]) + pure_python = True + + if (filename.endswith(".whl") and not is_wheel_platform_independent(filename)): + any_not_pure_python = True + pure_python = False + + # does this module matches any recipe name? + if mname.lower() in _requirement_names: + continue + + color = Out_Fore.GREEN if pure_python else Out_Fore.RED + ignored = "" if pure_python else ignored_str + + info( + f" {color}{mname}{Out_Fore.WHITE} : " + f"{Out_Style.BRIGHT}{mver}{Out_Style.RESET_ALL}" + f"{ignored}" + ) + + if pure_python: + processed_modules.append(f"{mname}=={mver}") + info(" ") + + if any_not_pure_python: + warning("Some packages were ignored because they are not pure Python.") + warning("To install the ignored packages, explicitly list them in your requirements file.") + + return processed_modules + + def run_pymodules_install(ctx, arch, modules, project_dir=None, ignore_setup_py=False): """ This function will take care of all non-recipe things, by: @@ -663,6 +774,10 @@ def run_pymodules_install(ctx, arch, modules, project_dir=None, info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE FOR ARCH: {} ***'.format(arch)) + # don't run process_python_modules in tests + if ctx.recipe_build_order.__class__.__name__ != "Mock": + modules = process_python_modules(ctx, modules) + modules = [m for m in modules if ctx.not_has_package(m, arch)] # We change current working directory later, so this has to be an absolute diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 9c086e0c83..32bc2ec2b0 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -979,7 +979,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env['LANG'] = "en_GB.UTF-8" # Binaries made by packages installed by pip - self.patch_shebangs(self._host_recipe.local_bin, self.real_hostpython_location) + self.patch_shebangs(self._host_recipe.local_bin, self._host_recipe.python_exe) env["PATH"] = self._host_recipe.local_bin + ":" + self._host_recipe.site_bin + ":" + env["PATH"] host_env = self.get_hostrecipe_env(arch) @@ -1022,10 +1022,9 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True): info('Installing {} into site-packages'.format(self.name)) - hostpython = sh.Command(self.hostpython_location) hpenv = env.copy() with current_directory(self.get_build_dir(arch.arch)): - shprint(hostpython, '-m', 'pip', 'install', '.', + shprint(self._host_recipe.pip, 'install', '.', '--compile', '--target', self.ctx.get_python_install_dir(arch.arch), _env=hpenv, *self.setup_extra_args @@ -1045,8 +1044,7 @@ def hostpython_site_dir(self): def install_hostpython_package(self, arch): env = self.get_hostrecipe_env(arch) - real_hostpython = sh.Command(self.real_hostpython_location) - shprint(real_hostpython, '-m', 'pip', 'install', '.', + shprint(self._host_recipe.pip, 'install', '.', '--compile', '--root={}'.format(self._host_recipe.site_root), _env=env, *self.setup_extra_args) @@ -1075,8 +1073,7 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True): pip_options.append("--upgrade") # Use system's pip pip_env = self.get_hostrecipe_env() - pip_env["HOME"] = "/tmp" - shprint(sh.Command(self.real_hostpython_location), "-m", "pip", *pip_options, _env=pip_env) + shprint(self._host_recipe.pip, *pip_options, _env=pip_env) def restore_hostpython_prerequisites(self, packages): _packages = [] @@ -1270,10 +1267,14 @@ def get_recipe_env(self, arch, **kwargs): return env def get_wheel_platform_tag(self, arch): + # https://peps.python.org/pep-0738/#packaging + # official python only supports 64 bit: + # android_21_arm64_v8a + # android_21_x86_64 return f"android_{self.ctx.ndk_api}_" + { - "armeabi-v7a": "arm", - "arm64-v8a": "aarch64", + "arm64-v8a": "arm64_v8a", "x86_64": "x86_64", + "armeabi-v7a": "arm", "x86": "i686", }[arch.arch] diff --git a/pythonforandroid/recipes/hostpython3/__init__.py b/pythonforandroid/recipes/hostpython3/__init__.py index afc4df4955..3e8ad78007 100644 --- a/pythonforandroid/recipes/hostpython3/__init__.py +++ b/pythonforandroid/recipes/hostpython3/__init__.py @@ -36,7 +36,7 @@ class HostPython3Recipe(Recipe): :class:`~pythonforandroid.python.HostPythonRecipe` ''' - version = '3.14.0' + version = '3.14.2' url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz' '''The default url to download our host python recipe. This url will @@ -113,6 +113,14 @@ def site_dir(self): f"usr/local/lib/python{p_version.major}.{p_version.minor}/site-packages/" ) + @property + def _pip(self): + return join(self.local_bin, "pip3") + + @property + def pip(self): + return sh.Command(self._pip) + def build_arch(self, arch): env = self.get_recipe_env(arch) @@ -160,10 +168,12 @@ def build_arch(self, arch): ensure_dir(self.site_root) self.ctx.hostpython = self.python_exe + if build_configured: + shprint( sh.Command(self.python_exe), "-m", "ensurepip", "--root", self.site_root, "-U", - _env={"HOME": "/tmp"} + _env={"HOME": "/tmp", "PATH": self.local_bin} ) diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index fb69b9a366..462847bc31 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -54,7 +54,7 @@ class Python3Recipe(TargetPythonRecipe): :class:`~pythonforandroid.python.GuestPythonRecipe` ''' - version = '3.14.0' + version = '3.14.2' _p_version = Version(version) url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz' name = 'python3' @@ -78,6 +78,7 @@ class Python3Recipe(TargetPythonRecipe): if _p_version.minor >= 14: patches.append('patches/3.14_armv7l_fix.patch') + patches.append('patches/3.14_fix_remote_debug.patch') if shutil.which('lld') is not None: if _p_version.minor == 7: diff --git a/pythonforandroid/recipes/python3/patches/3.14_fix_remote_debug.patch b/pythonforandroid/recipes/python3/patches/3.14_fix_remote_debug.patch new file mode 100644 index 0000000000..2b62c071db --- /dev/null +++ b/pythonforandroid/recipes/python3/patches/3.14_fix_remote_debug.patch @@ -0,0 +1,28 @@ +diff '--color=auto' -uNr cpython-3.14.2/Modules/_remote_debugging_module.c cpython-3.14.2.mod/Modules/_remote_debugging_module.c +--- cpython-3.14.2/Modules/_remote_debugging_module.c 2025-12-05 22:19:16.000000000 +0530 ++++ cpython-3.14.2.mod/Modules/_remote_debugging_module.c 2025-12-13 20:22:44.011497868 +0530 +@@ -812,7 +812,9 @@ + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } +-#elif defined(__linux__) ++ ++// https://github.com/python/cpython/commit/1963e701001839389cfb1b11d803b0743f4705d7 ++#elif defined(__linux__) && HAVE_PROCESS_VM_READV + // On Linux, search for asyncio debug in executable or DLL + address = search_linux_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); + if (address == 0) { +diff '--color=auto' -uNr cpython-3.14.2/Python/remote_debug.h cpython-3.14.2.mod/Python/remote_debug.h +--- cpython-3.14.2/Python/remote_debug.h 2025-12-05 22:19:16.000000000 +0530 ++++ cpython-3.14.2.mod/Python/remote_debug.h 2025-12-13 20:23:27.917518543 +0530 +@@ -881,7 +881,9 @@ + handle->pid); + _PyErr_ChainExceptions1(exc); + } +-#elif defined(__linux__) ++ ++// https://github.com/python/cpython/commit/1963e701001839389cfb1b11d803b0743f4705d7 ++#elif defined(__linux__) && HAVE_PROCESS_VM_READV + // On Linux, search for 'python' in executable or DLL + address = search_linux_map_for_section(handle, "PyRuntime", "python"); + if (address == 0) {