From 967e2cf164d1dfc1687034858c5db0b94bdddccd Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 29 Mar 2025 06:31:36 +0530 Subject: [PATCH 01/32] Support building without isolation --- pyodide_build/cli/build.py | 73 ++++++++++++++++++-- pyodide_build/out_of_tree/build.py | 11 ++- pyodide_build/pypabuild.py | 106 ++++++++++++++++++++++++++--- 3 files changed, 172 insertions(+), 18 deletions(-) diff --git a/pyodide_build/cli/build.py b/pyodide_build/cli/build.py index 10a3cb69..a312cc9a 100644 --- a/pyodide_build/cli/build.py +++ b/pyodide_build/cli/build.py @@ -46,6 +46,8 @@ def pypi( output_directory: Path, exports: str, config_settings: ConfigSettingsType, + isolation: bool = True, + skip_dependency_check: bool = False, ) -> Path: """Fetch a wheel from pypi, or build from source if none available.""" with tempfile.TemporaryDirectory() as tmpdir: @@ -65,6 +67,8 @@ def pypi( output_directory, convert_exports(exports), config_settings, + isolation=isolation, + skip_dependency_check=skip_dependency_check, ) return built_wheel @@ -86,6 +90,8 @@ def url( output_directory: Path, exports: str, config_settings: ConfigSettingsType, + isolation: bool = True, + skip_dependency_check: bool = False, ) -> Path: """Fetch a wheel or build sdist from url.""" with tempfile.TemporaryDirectory() as tmpdir: @@ -102,7 +108,12 @@ def url( # unzipped into subfolder builddir = files[0] wheel_path = build.run( - builddir, output_directory, convert_exports(exports), config_settings + builddir, + output_directory, + convert_exports(exports), + config_settings, + isolation=isolation, + skip_dependency_check=skip_dependency_check, ) return wheel_path @@ -112,10 +123,17 @@ def source( output_directory: Path, exports: str, config_settings: ConfigSettingsType, + isolation: bool = True, + skip_dependency_check: bool = False, ) -> Path: """Use pypa/build to build a Python package from source""" built_wheel = build.run( - source_location, output_directory, convert_exports(exports), config_settings + source_location, + output_directory, + convert_exports(exports), + config_settings, + isolation=isolation, + skip_dependency_check=skip_dependency_check, ) return built_wheel @@ -164,6 +182,19 @@ def main( compression_level: int = typer.Option( 6, help="Compression level to use for the created zip file" ), + no_isolation: bool = typer.Option( + False, + "--no-isolation", + "-n", + help="Disable building the project in an isolated virtual environment. " + "Build dependencies must be installed separately when this option is used", + ), + skip_dependency_check: bool = typer.Option( + False, + "--skip-dependency-check", + "-x", + help="Do not check that build dependencies are installed", + ), config_setting: Optional[list[str]] = typer.Option( # noqa: UP007 typer does not accept list[str] | None yet. None, "--config-setting", @@ -232,6 +263,8 @@ def main( # dependencies? Not sure this makes sense... convert_exports(exports), config_settings, + isolation=not no_isolation, + skip_dependency_check=skip_dependency_check, output_lockfile=output_lockfile, ) except BaseException as e: @@ -247,17 +280,43 @@ def main( source_location = source_location[0 : source_location.find("[")] if not source_location: # build the current folder - wheel = source(Path.cwd(), outpath, exports, config_settings) + wheel = source( + Path.cwd(), + outpath, + exports, + config_settings, + isolation=not no_isolation, + skip_dependency_check=skip_dependency_check, + ) elif source_location.find("://") != -1: - wheel = url(source_location, outpath, exports, config_settings) + wheel = url( + source_location, + outpath, + exports, + config_settings, + isolation=not no_isolation, + skip_dependency_check=skip_dependency_check, + ) elif Path(source_location).is_dir(): # a folder, build it wheel = source( - Path(source_location).resolve(), outpath, exports, config_settings + Path(source_location).resolve(), + outpath, + exports, + config_settings, + isolation=not no_isolation, + skip_dependency_check=skip_dependency_check, ) elif source_location.find("/") == -1: # try fetch or build from pypi - wheel = pypi(source_location, outpath, exports, config_settings) + wheel = pypi( + source_location, + outpath, + exports, + config_settings, + isolation=not no_isolation, + skip_dependency_check=skip_dependency_check, + ) else: raise RuntimeError(f"Couldn't determine source type for {source_location}") # now build deps for wheel @@ -271,6 +330,8 @@ def main( # dependencies? Not sure this makes sense... convert_exports(exports), config_settings, + isolation=not no_isolation, + skip_dependency_check=skip_dependency_check, output_lockfile=output_lockfile, compression_level=compression_level, ) diff --git a/pyodide_build/out_of_tree/build.py b/pyodide_build/out_of_tree/build.py index 6d450849..951111ef 100644 --- a/pyodide_build/out_of_tree/build.py +++ b/pyodide_build/out_of_tree/build.py @@ -13,6 +13,8 @@ def run( outdir: Path, exports: _BuildSpecExports, config_settings: ConfigSettingsType, + isolation: bool = True, + skip_dependency_check: bool = False, ) -> Path: outdir = outdir.resolve() cflags = build_env.get_build_flag("SIDE_MODULE_CFLAGS") @@ -39,7 +41,14 @@ def run( ) with build_env_ctx as env: - built_wheel = pypabuild.build(srcdir, outdir, env, config_settings) + built_wheel = pypabuild.build( + srcdir, + outdir, + env, + config_settings, + isolation=isolation, + skip_dependency_check=skip_dependency_check, + ) wheel_path = Path(built_wheel) if "emscripten" in wheel_path.name: diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index 392b0c09..a43f071a 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -5,6 +5,16 @@ import sys import traceback from collections.abc import Callable, Iterator, Mapping, Sequence + + +# A helper function from pypa/build/__main__.py since +# it's not in the vendorized code we have +# TODO: we should move this to a new file. it's out of place +# between the other imports ;-) +def _format_dep_chain(dep_chain: Sequence[str]) -> str: + return " -> ".join(dep.partition(";")[0].strip() for dep in dep_chain) + + from contextlib import contextmanager from itertools import chain from pathlib import Path @@ -54,7 +64,7 @@ def _gen_runner( cross_build_env: Mapping[str, str], - isolated_build_env: _DefaultIsolatedEnv, + isolated_build_env: _DefaultIsolatedEnv = None, ) -> Callable[[Sequence[str], str | None, Mapping[str, str] | None], None]: """ This returns a slightly modified version of default subprocess runner that pypa/build uses. @@ -79,7 +89,15 @@ def _runner(cmd, cwd=None, extra_environ=None): # Some build dependencies like cmake, meson installs binaries to this directory # and we should add it to the PATH so that they can be found. - env["BUILD_ENV_SCRIPTS_DIR"] = isolated_build_env.scripts_dir + if isolated_build_env: + env["BUILD_ENV_SCRIPTS_DIR"] = isolated_build_env.scripts_dir + else: + # For non-isolated builds, set a fallback path or use the current Python path + import sysconfig + + scripts_dir = sysconfig.get_path("scripts") + env["BUILD_ENV_SCRIPTS_DIR"] = scripts_dir + env["PATH"] = f"{cross_build_env['COMPILER_WRAPPER_DIR']}:{env['PATH']}" # For debugging: Uncomment the following line to print the build command # print("Build backend call:", " ".join(str(x) for x in cmd), file=sys.stderr) @@ -191,6 +209,45 @@ def _build_in_isolated_env( ) +def _build_in_current_env( + build_env: Mapping[str, str], + srcdir: Path, + outdir: str, + distribution: Literal["sdist", "wheel"], + config_settings: ConfigSettingsType, + skip_dependency_check: bool = False, +) -> str: + with common.replace_env(build_env): + # Setup sysconfigdata path in environment + sysconfigdata_name = get_build_flag("SYSCONFIG_NAME") + sysconfig_dir = Path(get_build_flag("TARGETINSTALLDIR")) / "sysconfigdata" + env = os.environ.copy() + env["_PYTHON_SYSCONFIGDATA_NAME"] = sysconfigdata_name + env["PYTHONPATH"] = f"{str(sysconfig_dir)}:{env.get('PYTHONPATH', '')}" + + with common.replace_env(env): + builder = _ProjectBuilder(srcdir, runner=_gen_runner(build_env)) + + if not skip_dependency_check: + missing = builder.check_dependencies( + distribution, config_settings or {} + ) + if missing: + dependencies = "".join( + "\n\t" + dep + for deps in missing + for dep in (deps[0], _format_dep_chain(deps[1:])) + if dep + ) + _error(f"Missing dependencies:{dependencies}") + + return builder.build( + distribution, + outdir, + config_settings, + ) + + def parse_backend_flags(backend_flags: str | list[str]) -> ConfigSettingsType: config_settings: dict[str, str | list[str]] = {} @@ -254,7 +311,9 @@ def make_command_wrapper_symlinks(symlink_dir: Path) -> dict[str, str]: @contextmanager -def _create_symlink_dir(env: dict[str, str], build_dir: Path | None): +def _create_symlink_dir( + env: dict[str, str], build_dir: Path | None, no_isolation: bool = False +): if build_dir: # If we're running under build-recipes, leave the symlinks in # the build directory. This helps with reproducing. @@ -264,10 +323,22 @@ def _create_symlink_dir(env: dict[str, str], build_dir: Path | None): yield symlink_dir return - # Running from "pyodide build". Put symlinks in a temporary directory. - # TODO: Add a debug option to save the symlinks. - with TemporaryDirectory() as symlink_dir_str: - yield Path(symlink_dir_str) + # TODO: FIXME: compiler wrappers are still ending up in a temporary + # directory, which breaks persistent builds. This is not ideal, but + # it is better than nothing so this is non-blocking for now. It has + # to be investigated further. + + if no_isolation: + # For non-isolated builds, create a persistent directory in the current working directory + # or in a well-known location like ~/.pyodide/compiler_wrappers + symlink_dir = Path.cwd() / ".pyodide_compiler_wrappers" + symlink_dir.mkdir(exist_ok=True) + yield symlink_dir + else: + # Running from "pyodide build". Put symlinks in a temporary directory. + # TODO: Add a debug option to save the symlinks. + with TemporaryDirectory() as symlink_dir_str: + yield Path(symlink_dir_str) @contextmanager @@ -281,6 +352,7 @@ def get_build_env( target_install_dir: str, exports: _BuildSpecExports, build_dir: Path | None = None, + no_isolation: bool = False, ) -> Iterator[dict[str, str]]: """ Returns a dict of environment variables that should be used when building @@ -299,7 +371,7 @@ def get_build_env( args["exports"] = exports env = env.copy() - with _create_symlink_dir(env, build_dir) as symlink_dir: + with _create_symlink_dir(env, build_dir, no_isolation) as symlink_dir: env.update(make_command_wrapper_symlinks(symlink_dir)) sysconfig_dir = Path(get_build_flag("TARGETINSTALLDIR")) / "sysconfigdata" args["PYTHONPATH"] = sys.path + [str(symlink_dir), str(sysconfig_dir)] @@ -328,12 +400,24 @@ def build( outdir: Path, build_env: Mapping[str, str], config_settings: ConfigSettingsType, + isolation: bool = True, + skip_dependency_check: bool = False, ) -> str: try: with _handle_build_error(): - built = _build_in_isolated_env( - build_env, srcdir, str(outdir), "wheel", config_settings - ) + if isolation: + built = _build_in_isolated_env( + build_env, srcdir, str(outdir), "wheel", config_settings + ) + else: + built = _build_in_current_env( + build_env, + srcdir, + str(outdir), + "wheel", + config_settings, + skip_dependency_check, + ) print("{bold}{green}Successfully built {}{reset}".format(built, **_STYLES)) return built except Exception as e: # pragma: no cover From f7968106b0e30c3b0f06c55dd46e753d9f3eea84 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 30 Mar 2025 05:46:08 +0530 Subject: [PATCH 02/32] Pass along extra args, fix tests [integration] --- pyodide_build/out_of_tree/pypi.py | 34 +++++++++++++++++++++++-------- pyodide_build/tests/test_cli.py | 27 +++++++++++++++++++++--- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/pyodide_build/out_of_tree/pypi.py b/pyodide_build/out_of_tree/pypi.py index a5750b1d..a4c58153 100644 --- a/pyodide_build/out_of_tree/pypi.py +++ b/pyodide_build/out_of_tree/pypi.py @@ -70,12 +70,12 @@ def stream_redirected(to=os.devnull, stream=None): to = None -def get_built_wheel(url): - return _get_built_wheel_internal(url)["path"] +def get_built_wheel(url, isolation=True, skip_dependency_check=False): + return _get_built_wheel_internal(url, isolation, skip_dependency_check)["path"] @cache -def _get_built_wheel_internal(url): +def _get_built_wheel_internal(url, isolation=True, skip_dependency_check=False): parsed_url = urlparse(url) gz_name = Path(parsed_url.path).name @@ -107,6 +107,8 @@ def _get_built_wheel_internal(url): build_path / "dist", PyPIProvider.BUILD_EXPORTS, PyPIProvider.BUILD_FLAGS, + isolation=isolation, + skip_dependency_check=skip_dependency_check, ) except BaseException as e: logger.error(" Failed\n Error is:") @@ -199,11 +201,15 @@ def get_project_from_pypi(package_name, extras): def download_or_build_wheel( - url: str, target_directory: Path, compression_level: int = 6 + url: str, + target_directory: Path, + compression_level: int = 6, + isolation: bool = True, + skip_dependency_check: bool = False, ) -> None: parsed_url = urlparse(url) if parsed_url.path.endswith("gz"): - wheel_file = get_built_wheel(url) + wheel_file = get_built_wheel(url, isolation, skip_dependency_check) shutil.copy(wheel_file, target_directory) wheel_path = target_directory / wheel_file.name elif parsed_url.path.endswith(".whl"): @@ -333,6 +339,8 @@ def _resolve_and_build( build_dependencies: bool, extras: list[str], output_lockfile: str | None, + isolation: bool = True, + skip_dependency_check: bool = False, compression_level: int = 6, ) -> None: requirements = [] @@ -362,7 +370,7 @@ def _resolve_and_build( if output_lockfile is not None and len(output_lockfile) > 0: version_file = open(output_lockfile, "w") for x in result.mapping.values(): - download_or_build_wheel(x.url, target_folder) + download_or_build_wheel(x.url, target_folder, compression_level) if len(x.extras) > 0: extratxt = "[" + ",".join(x.extras) + "]" else: @@ -380,7 +388,10 @@ def build_wheels_from_pypi_requirements( skip_dependency: list[str], exports: _BuildSpecExports, config_settings: ConfigSettingsType, - output_lockfile: str | None, + isolation: bool = True, + skip_dependency_check: bool = False, + output_lockfile: str | None = None, + compression_level: int = 6, ) -> None: """ Given a list of package requirements, build or fetch them. If build_dependencies is true, then @@ -395,6 +406,9 @@ def build_wheels_from_pypi_requirements( build_dependencies, extras=[], output_lockfile=output_lockfile, + isolation=isolation, + skip_dependency_check=skip_dependency_check, + compression_level=compression_level, ) @@ -404,7 +418,9 @@ def build_dependencies_for_wheel( skip_dependency: list[str], exports: _BuildSpecExports, config_settings: ConfigSettingsType, - output_lockfile: str | None, + isolation: bool = True, + skip_dependency_check: bool = False, + output_lockfile: str | None = None, compression_level: int = 6, ) -> None: """Extract dependencies from this wheel and build pypi dependencies @@ -435,6 +451,8 @@ def build_dependencies_for_wheel( build_dependencies=True, extras=extras, output_lockfile=output_lockfile, + isolation=isolation, + skip_dependency_check=skip_dependency_check, compression_level=compression_level, ) # add the current wheel to the package-versions.txt diff --git a/pyodide_build/tests/test_cli.py b/pyodide_build/tests/test_cli.py index 09e534a1..f3f527cd 100644 --- a/pyodide_build/tests/test_cli.py +++ b/pyodide_build/tests/test_cli.py @@ -355,7 +355,14 @@ def test_py_compile(tmp_path, target, compression_level): def test_build1(tmp_path, monkeypatch, dummy_xbuildenv, mock_emscripten): from pyodide_build import pypabuild - def mocked_build(srcdir: Path, outdir: Path, env: Any, backend_flags: Any) -> str: + def mocked_build( + srcdir: Path, + outdir: Path, + env: Any, + backend_flags: Any, + isolation=True, + skip_dependency_check=False, + ) -> str: results["srcdir"] = srcdir results["outdir"] = outdir results["backend_flags"] = backend_flags @@ -431,7 +438,14 @@ def unpack_archive_shim(*args): exports_ = None - def run_shim(builddir, output_directory, exports, backend_flags): + def run_shim( + builddir, + output_directory, + exports, + backend_flags, + isolation=True, + skip_dependency_check=False, + ): nonlocal exports_ exports_ = exports @@ -491,7 +505,14 @@ def test_build_config_settings(monkeypatch, dummy_xbuildenv): config_settings_passed = None - def run(srcdir, outdir, exports, config_settings): + def run( + srcdir, + outdir, + exports, + config_settings, + isolation=True, + skip_dependency_check=False, + ): nonlocal config_settings_passed config_settings_passed = config_settings From c2ae4af21d2f4cf68d6140a8852870206489b19a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 30 Mar 2025 06:26:08 +0530 Subject: [PATCH 03/32] Add a few stub tests --- pyodide_build/tests/test_cli.py | 198 ++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/pyodide_build/tests/test_cli.py b/pyodide_build/tests/test_cli.py index f3f527cd..b3a86873 100644 --- a/pyodide_build/tests/test_cli.py +++ b/pyodide_build/tests/test_cli.py @@ -678,3 +678,201 @@ def test_build_constraint(tmp_path, dummy_xbuildenv, mock_emscripten, capsys): build_dir = RECIPE_DIR / pkg / "build" assert (build_dir / "setuptools.version").read_text() == "74.1.3" assert (build_dir / "pytest.version").read_text() == "7.0.0" + + +@pytest.mark.parametrize( + "isolation_flag", + [ + None, + "--no-isolation", + ], +) +def test_build_isolation_flags( + tmp_path, monkeypatch, dummy_xbuildenv, mock_emscripten, isolation_flag +): + """Test that build works with different isolation flags.""" + from pyodide_build import pypabuild + + build_calls = [] + + def mocked_build( + srcdir, + outdir, + env, + config_settings, + isolation=True, + skip_dependency_check=False, + ): + build_calls.append( + { + "srcdir": srcdir, + "isolation": isolation, + "skip_dependency_check": skip_dependency_check, + } + ) + dummy_wheel = outdir / "package-1.0.0-py3-none-any.whl" + return str(dummy_wheel) + + monkeypatch.setattr(pypabuild, "build", mocked_build) + monkeypatch.setattr(build_env, "check_emscripten_version", lambda: None) + monkeypatch.setattr(build_env, "replace_so_abi_tags", lambda whl: None) + monkeypatch.setattr( + common, "retag_wheel", lambda wheel_path, platform: Path(wheel_path) + ) + + from contextlib import nullcontext + + monkeypatch.setattr(common, "modify_wheel", lambda whl: nullcontext()) + + app = typer.Typer() + app.command(**build.main.typer_kwargs)(build.main) # type:ignore[attr-defined] + + srcdir = tmp_path / "in" + outdir = tmp_path / "out" + srcdir.mkdir() + + args = [str(srcdir), "--outdir", str(outdir)] + if isolation_flag: + args.append(isolation_flag) + + result = runner.invoke(app, args) + + assert result.exit_code == 0, result.stdout + assert len(build_calls) == 1 + + # Check that isolation was properly passed + expected_isolation = isolation_flag is None + assert build_calls[0]["isolation"] == expected_isolation + + +@pytest.mark.parametrize( + "skip_check_flag", + [ + None, # Default: with dependency check + "--skip-dependency-check", + "-x", + ], +) +def test_build_skip_dependency_check( + tmp_path, monkeypatch, dummy_xbuildenv, mock_emscripten, skip_check_flag +): + """Test that build works with different skip dependency check flags.""" + from pyodide_build import pypabuild + + build_calls = [] + + def mocked_build( + srcdir, + outdir, + env, + config_settings, + isolation=True, + skip_dependency_check=False, + ): + build_calls.append( + { + "srcdir": srcdir, + "isolation": isolation, + "skip_dependency_check": skip_dependency_check, + } + ) + dummy_wheel = outdir / "package-1.0.0-py3-none-any.whl" + return str(dummy_wheel) + + monkeypatch.setattr(pypabuild, "build", mocked_build) + monkeypatch.setattr(build_env, "check_emscripten_version", lambda: None) + monkeypatch.setattr(build_env, "replace_so_abi_tags", lambda whl: None) + monkeypatch.setattr( + common, "retag_wheel", lambda wheel_path, platform: Path(wheel_path) + ) + + from contextlib import nullcontext + + monkeypatch.setattr(common, "modify_wheel", lambda whl: nullcontext()) + + app = typer.Typer() + app.command(**build.main.typer_kwargs)(build.main) # type:ignore[attr-defined] + + srcdir = tmp_path / "in" + outdir = tmp_path / "out" + srcdir.mkdir() + + args = [str(srcdir), "--outdir", str(outdir)] + if skip_check_flag: + args.append(skip_check_flag) + + result = runner.invoke(app, args) + + assert result.exit_code == 0, result.stdout + assert len(build_calls) == 1 + + # Check that skip_dependency_check was properly passed + expected_skip = skip_check_flag is not None + assert build_calls[0]["skip_dependency_check"] == expected_skip + + +@pytest.mark.parametrize( + "isolation,skip_check", + [ + (True, False), # default: with isolation, without skipping dependency checking + (False, False), + (True, True), + (False, True), + ], +) +def test_build_combined_flags( + tmp_path, monkeypatch, dummy_xbuildenv, mock_emscripten, isolation, skip_check +): + """Test combinations of isolation and skip dependency check flags.""" + from pyodide_build import pypabuild + + build_calls = [] + + def mocked_build( + srcdir, + outdir, + env, + config_settings, + isolation=True, + skip_dependency_check=False, + ): + build_calls.append( + { + "srcdir": srcdir, + "isolation": isolation, + "skip_dependency_check": skip_dependency_check, + } + ) + dummy_wheel = outdir / "package-1.0.0-py3-none-any.whl" + return str(dummy_wheel) + + monkeypatch.setattr(pypabuild, "build", mocked_build) + monkeypatch.setattr(build_env, "check_emscripten_version", lambda: None) + monkeypatch.setattr(build_env, "replace_so_abi_tags", lambda whl: None) + monkeypatch.setattr( + common, "retag_wheel", lambda wheel_path, platform: Path(wheel_path) + ) + + from contextlib import nullcontext + + monkeypatch.setattr(common, "modify_wheel", lambda whl: nullcontext()) + + app = typer.Typer() + app.command(**build.main.typer_kwargs)(build.main) # type:ignore[attr-defined] + + srcdir = tmp_path / "in" + outdir = tmp_path / "out" + srcdir.mkdir() + + args = [str(srcdir), "--outdir", str(outdir)] + if not isolation: + args.append("--no-isolation") + if skip_check: + args.append("--skip-dependency-check") + + result = runner.invoke(app, args) + + assert result.exit_code == 0, result.stdout + assert len(build_calls) == 1 + assert build_calls[0]["isolation"] == isolation + assert build_calls[0]["skip_dependency_check"] == skip_check From b52ea77b0df33e7653906148b98f9d5013f4c91b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 30 Mar 2025 06:39:41 +0530 Subject: [PATCH 04/32] Add [integration] test to build NumPy with `-nx` --- .github/workflows/main.yml | 1 + integration_tests/Makefile | 13 +++++++++++++ integration_tests/src/numpy.sh | 13 ++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dca6d7e6..5b3a5d2b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,6 +98,7 @@ jobs: {name: test-recipe, installer: uv}, {name: test-src, installer: uv}, {name: test-integration-marker, installer: pip}, # installer doesn't matter + {name: test-src-no-isolation, installer: pip}, # installer doesn't matter ] os: [ubuntu-latest, macos-latest] if: needs.check-integration-test-trigger.outputs.run-integration-test diff --git a/integration_tests/Makefile b/integration_tests/Makefile index d50dc9be..68c3a503 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -23,6 +23,19 @@ test-src: check @echo "... Passed" +.PHONY: test-src-no-isolation +test-src-no-isolation: check + @echo "... Running integration tests for building src with --no-isolation --skip-dependency-check" + + # Ensure the necessary build dependencies are installed in the + # host environment as we need to do so ourselves. + + pip install meson-python meson cython + + ./src/numpy.sh --no-isolation --skip-dependency-check + + @echo "... Passed" + .PHONY: check check: @echo "... Checking dependencies" diff --git a/integration_tests/src/numpy.sh b/integration_tests/src/numpy.sh index 1abe6b44..8c790a8f 100755 --- a/integration_tests/src/numpy.sh +++ b/integration_tests/src/numpy.sh @@ -2,6 +2,17 @@ set -e +NO_ISOLATION="" +SKIP_DEPS="" + +while [[ "$#" -gt 0 ]]; do + case $1 in + --no-isolation) NO_ISOLATION="--no-isolation"; shift ;; + --skip-dependency-check) SKIP_DEPS="--skip-dependency-check"; shift ;; + *) echo "Unknown parameter: $1"; exit 1 ;; + esac +done + VERSION="2.0.2" URL="https://files.pythonhosted.org/packages/source/n/numpy/numpy-${VERSION}.tar.gz" @@ -10,4 +21,4 @@ tar -xf numpy-${VERSION}.tar.gz cd numpy-${VERSION} MESON_CROSS_FILE=$(pyodide config get meson_cross_file) -${UV_RUN_PREFIX} pyodide build -Csetup-args=-Dallow-noblas=true -Csetup-args=--cross-file="${MESON_CROSS_FILE}" +${UV_RUN_PREFIX} pyodide build -Csetup-args=-Dallow-noblas=true -Csetup-args=--cross-file="${MESON_CROSS_FILE}" $NO_ISOLATION $SKIP_DEPS From 4515ef7d4ce49de490eb8da50bba7252c6f0f7a1 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 30 Mar 2025 06:41:33 +0530 Subject: [PATCH 05/32] Don't run tests on pushes with PRs --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b3a5d2b..629319bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,8 @@ name: CI on: push: + branches: + - main pull_request: concurrency: From 523e3d31de4e622de6e52008c915a8e18780d26c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 30 Mar 2025 06:41:49 +0530 Subject: [PATCH 06/32] Trigger [integration] tests From ed6175995cedc839ee4b8ff95928b95c45132534 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 30 Mar 2025 06:42:56 +0530 Subject: [PATCH 07/32] Drop [integration] no-isolation test on macOS --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 629319bc..991dfd90 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -100,9 +100,11 @@ jobs: {name: test-recipe, installer: uv}, {name: test-src, installer: uv}, {name: test-integration-marker, installer: pip}, # installer doesn't matter - {name: test-src-no-isolation, installer: pip}, # installer doesn't matter ] os: [ubuntu-latest, macos-latest] + include: + - task: {name: test-src-no-isolation, installer: pip} # installer doesn't matter + os: ubuntu-latest if: needs.check-integration-test-trigger.outputs.run-integration-test steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 From 8530171ac2f30bbb18099089382cffb89ca7c51d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 05:00:05 +0530 Subject: [PATCH 08/32] No need to determine `sysconfigdata` again --- pyodide_build/pypabuild.py | 51 +++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index a43f071a..e400c466 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -209,6 +209,16 @@ def _build_in_isolated_env( ) +# TODO: move to common.py +def _format_missing_dependencies(missing) -> str: + return "".join( + "\n\t" + dep + for deps in missing + for dep in (deps[0], _format_dep_chain(deps[1:])) + if dep + ) + + def _build_in_current_env( build_env: Mapping[str, str], srcdir: Path, @@ -218,34 +228,19 @@ def _build_in_current_env( skip_dependency_check: bool = False, ) -> str: with common.replace_env(build_env): - # Setup sysconfigdata path in environment - sysconfigdata_name = get_build_flag("SYSCONFIG_NAME") - sysconfig_dir = Path(get_build_flag("TARGETINSTALLDIR")) / "sysconfigdata" - env = os.environ.copy() - env["_PYTHON_SYSCONFIGDATA_NAME"] = sysconfigdata_name - env["PYTHONPATH"] = f"{str(sysconfig_dir)}:{env.get('PYTHONPATH', '')}" - - with common.replace_env(env): - builder = _ProjectBuilder(srcdir, runner=_gen_runner(build_env)) - - if not skip_dependency_check: - missing = builder.check_dependencies( - distribution, config_settings or {} - ) - if missing: - dependencies = "".join( - "\n\t" + dep - for deps in missing - for dep in (deps[0], _format_dep_chain(deps[1:])) - if dep - ) - _error(f"Missing dependencies:{dependencies}") - - return builder.build( - distribution, - outdir, - config_settings, - ) + builder = _ProjectBuilder(srcdir, runner=_gen_runner(build_env)) + + if not skip_dependency_check: + missing = builder.check_dependencies(distribution, config_settings or {}) + if missing: + dependencies = _format_missing_dependencies(missing) + _error(f"Missing dependencies: {dependencies}") + + return builder.build( + distribution, + outdir, + config_settings, + ) def parse_backend_flags(backend_flags: str | list[str]) -> ConfigSettingsType: From e38a704953522fcdc7366f612d52b4c4e77e798b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 05:03:30 +0530 Subject: [PATCH 09/32] Accept any args to `numpy.sh` --- integration_tests/src/numpy.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/integration_tests/src/numpy.sh b/integration_tests/src/numpy.sh index 8c790a8f..465052b8 100755 --- a/integration_tests/src/numpy.sh +++ b/integration_tests/src/numpy.sh @@ -2,17 +2,6 @@ set -e -NO_ISOLATION="" -SKIP_DEPS="" - -while [[ "$#" -gt 0 ]]; do - case $1 in - --no-isolation) NO_ISOLATION="--no-isolation"; shift ;; - --skip-dependency-check) SKIP_DEPS="--skip-dependency-check"; shift ;; - *) echo "Unknown parameter: $1"; exit 1 ;; - esac -done - VERSION="2.0.2" URL="https://files.pythonhosted.org/packages/source/n/numpy/numpy-${VERSION}.tar.gz" @@ -21,4 +10,4 @@ tar -xf numpy-${VERSION}.tar.gz cd numpy-${VERSION} MESON_CROSS_FILE=$(pyodide config get meson_cross_file) -${UV_RUN_PREFIX} pyodide build -Csetup-args=-Dallow-noblas=true -Csetup-args=--cross-file="${MESON_CROSS_FILE}" $NO_ISOLATION $SKIP_DEPS +${UV_RUN_PREFIX} pyodide build -Csetup-args=-Dallow-noblas=true -Csetup-args=--cross-file="${MESON_CROSS_FILE}" "$@" From 0aed06b6b02ba02dd08d9686a7c00d4e54437129 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 05:03:58 +0530 Subject: [PATCH 10/32] Move helper functions to `pyodide_build.common` --- pyodide_build/common.py | 13 +++++++++++++ pyodide_build/pypabuild.py | 22 +--------------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/pyodide_build/common.py b/pyodide_build/common.py index fa0448b0..357655c6 100644 --- a/pyodide_build/common.py +++ b/pyodide_build/common.py @@ -315,6 +315,19 @@ def _get_sha256_checksum(archive: Path) -> str: return h.hexdigest() +def _format_dep_chain(dep_chain: Sequence[str]) -> str: + return " -> ".join(dep.partition(";")[0].strip() for dep in dep_chain) + + +def _format_missing_dependencies(missing) -> str: + return "".join( + "\n\t" + dep + for deps in missing + for dep in (deps[0], _format_dep_chain(deps[1:])) + if dep + ) + + def unpack_wheel(wheel_path: Path, target_dir: Path | None = None) -> None: if target_dir is None: target_dir = wheel_path.parent diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index e400c466..6450cb46 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -5,16 +5,6 @@ import sys import traceback from collections.abc import Callable, Iterator, Mapping, Sequence - - -# A helper function from pypa/build/__main__.py since -# it's not in the vendorized code we have -# TODO: we should move this to a new file. it's out of place -# between the other imports ;-) -def _format_dep_chain(dep_chain: Sequence[str]) -> str: - return " -> ".join(dep.partition(";")[0].strip() for dep in dep_chain) - - from contextlib import contextmanager from itertools import chain from pathlib import Path @@ -209,16 +199,6 @@ def _build_in_isolated_env( ) -# TODO: move to common.py -def _format_missing_dependencies(missing) -> str: - return "".join( - "\n\t" + dep - for deps in missing - for dep in (deps[0], _format_dep_chain(deps[1:])) - if dep - ) - - def _build_in_current_env( build_env: Mapping[str, str], srcdir: Path, @@ -233,7 +213,7 @@ def _build_in_current_env( if not skip_dependency_check: missing = builder.check_dependencies(distribution, config_settings or {}) if missing: - dependencies = _format_missing_dependencies(missing) + dependencies = common._format_missing_dependencies(missing) _error(f"Missing dependencies: {dependencies}") return builder.build( From 6c5da50434a00862e3e7d732504d62780fc6c3f8 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 05:05:17 +0530 Subject: [PATCH 11/32] Mention effect of `--skip-dependency-check` --- pyodide_build/cli/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyodide_build/cli/build.py b/pyodide_build/cli/build.py index a312cc9a..b55e610c 100644 --- a/pyodide_build/cli/build.py +++ b/pyodide_build/cli/build.py @@ -193,7 +193,8 @@ def main( False, "--skip-dependency-check", "-x", - help="Do not check that build dependencies are installed", + help="Do not check that the build dependencies are installed. This option " + "is only useful when used with --no-isolation.", ), config_setting: Optional[list[str]] = typer.Option( # noqa: UP007 typer does not accept list[str] | None yet. None, From 8325e803d78cc3e3833edceeb652aee2e7974891 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 05:45:27 +0530 Subject: [PATCH 12/32] Save symlinks in the build directory Co-Authored-By: Gyeongjae Choi --- pyodide_build/out_of_tree/build.py | 4 ++++ pyodide_build/pypabuild.py | 33 ++++++------------------------ 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/pyodide_build/out_of_tree/build.py b/pyodide_build/out_of_tree/build.py index 951111ef..400432a1 100644 --- a/pyodide_build/out_of_tree/build.py +++ b/pyodide_build/out_of_tree/build.py @@ -30,6 +30,9 @@ def run( env = os.environ.copy() env.update(build_env.get_build_environment_vars(get_pyodide_root())) + # create a persistent build dir in the source dir + build_dir = srcdir / ".pyodide_build" + build_dir.mkdir(exist_ok=True) build_env_ctx = pypabuild.get_build_env( env=env, pkgname="", @@ -38,6 +41,7 @@ def run( ldflags=ldflags, target_install_dir=target_install_dir, exports=exports, + build_dir=build_dir, ) with build_env_ctx as env: diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index 6450cb46..77eff0de 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -8,7 +8,6 @@ from contextlib import contextmanager from itertools import chain from pathlib import Path -from tempfile import TemporaryDirectory from typing import Literal, cast from build import BuildBackendException, ConfigSettingsType @@ -287,33 +286,13 @@ def make_command_wrapper_symlinks(symlink_dir: Path) -> dict[str, str]: @contextmanager def _create_symlink_dir( - env: dict[str, str], build_dir: Path | None, no_isolation: bool = False + env: dict[str, str], build_dir: Path, no_isolation: bool = False ): - if build_dir: - # If we're running under build-recipes, leave the symlinks in - # the build directory. This helps with reproducing. - symlink_dir = build_dir / "pywasmcross_symlinks" - shutil.rmtree(symlink_dir, ignore_errors=True) - symlink_dir.mkdir() - yield symlink_dir - return - - # TODO: FIXME: compiler wrappers are still ending up in a temporary - # directory, which breaks persistent builds. This is not ideal, but - # it is better than nothing so this is non-blocking for now. It has - # to be investigated further. - - if no_isolation: - # For non-isolated builds, create a persistent directory in the current working directory - # or in a well-known location like ~/.pyodide/compiler_wrappers - symlink_dir = Path.cwd() / ".pyodide_compiler_wrappers" - symlink_dir.mkdir(exist_ok=True) - yield symlink_dir - else: - # Running from "pyodide build". Put symlinks in a temporary directory. - # TODO: Add a debug option to save the symlinks. - with TemporaryDirectory() as symlink_dir_str: - yield Path(symlink_dir_str) + # Leave the symlinks in the build directory. This helps with reproducing. + symlink_dir = build_dir / "pywasmcross_symlinks" + shutil.rmtree(symlink_dir, ignore_errors=True) + symlink_dir.mkdir() + yield symlink_dir @contextmanager From efee932d5404f1939d46f15578eb8581f7461578 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 05:46:57 +0530 Subject: [PATCH 13/32] Git-ignore the created symlinks directory --- pyodide_build/out_of_tree/build.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pyodide_build/out_of_tree/build.py b/pyodide_build/out_of_tree/build.py index 400432a1..ca6f2e4d 100644 --- a/pyodide_build/out_of_tree/build.py +++ b/pyodide_build/out_of_tree/build.py @@ -1,5 +1,6 @@ import os from pathlib import Path +from textwrap import dedent from build import ConfigSettingsType @@ -33,6 +34,16 @@ def run( # create a persistent build dir in the source dir build_dir = srcdir / ".pyodide_build" build_dir.mkdir(exist_ok=True) + # don't track the build dir (helps if building in a git repo) + gitignore_path = build_dir / ".gitignore" + if not gitignore_path.exists(): + with gitignore_path.open("w") as f: + f.write( + dedent("""\ + # Created by pyodide-build + *""") + ) + build_env_ctx = pypabuild.get_build_env( env=env, pkgname="", From 8d17f99aa5098f24f876676c8c4e556a05ed64d8 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 05:47:27 +0530 Subject: [PATCH 14/32] Trigger [integration] tests From 97e41b693c3b29a487fc8419b662dfd3bd2fde34 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 06:16:50 +0530 Subject: [PATCH 15/32] Trigger [integration] tests From 47d458bb82640be852fa7acc0b20d8ef9149428c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 06:38:55 +0530 Subject: [PATCH 16/32] Fix test --- pyodide_build/tests/test_pypabuild.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyodide_build/tests/test_pypabuild.py b/pyodide_build/tests/test_pypabuild.py index 02bab233..5c6af469 100644 --- a/pyodide_build/tests/test_pypabuild.py +++ b/pyodide_build/tests/test_pypabuild.py @@ -77,6 +77,7 @@ def test_get_build_env(tmp_path, dummy_xbuildenv): ldflags="", target_install_dir=str(tmp_path), exports="pyinit", + build_dir=tmp_path, ) with build_env_ctx as env: From 4f793fdb6bac38cbf6613fbe4708d0b3e6d42b25 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 07:26:51 +0530 Subject: [PATCH 17/32] Add better "no isolation" [integration] test --- integration_tests/Makefile | 24 ++++++++++----- integration_tests/src/numpy_no_isolation.sh | 33 +++++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) create mode 100755 integration_tests/src/numpy_no_isolation.sh diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 68c3a503..f77e859c 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -27,15 +27,21 @@ test-src: check test-src-no-isolation: check @echo "... Running integration tests for building src with --no-isolation --skip-dependency-check" - # Ensure the necessary build dependencies are installed in the - # host environment as we need to do so ourselves. - - pip install meson-python meson cython - - ./src/numpy.sh --no-isolation --skip-dependency-check + # Some virtualenv workarounds from https://stackoverflow.com/a/24736236 + # to make sure that we are using the right environment + @( \ + set -e; \ + python -m venv ./test_venv; \ + source ./test_venv/bin/activate; \ + pip install meson-python meson cython; \ + pip install -e ../; \ + ./src/numpy_no_isolation.sh; \ + deactivate; \ + ) + + @rm -rf ./test_venv @echo "... Passed" - .PHONY: check check: @echo "... Checking dependencies" @@ -49,5 +55,9 @@ check: clean: rm -rf .pyodide-xbuildenv* rm -rf recipes/*/build + rm -rf test_venv rm -rf src/numpy-* + rm -rf src/numpy-*.tar.gz + rm -rf numpy-* + rm -rf numpy-*.tar.gz rm -rf dist diff --git a/integration_tests/src/numpy_no_isolation.sh b/integration_tests/src/numpy_no_isolation.sh new file mode 100755 index 00000000..8fc2b69b --- /dev/null +++ b/integration_tests/src/numpy_no_isolation.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# The same as "numpy.sh", but without the isolation. + +set -e + +VERSION="2.0.2" +URL="https://files.pythonhosted.org/packages/source/n/numpy/numpy-${VERSION}.tar.gz" + +wget $URL +tar -xf numpy-${VERSION}.tar.gz +cd numpy-${VERSION} + +MESON_CROSS_FILE=$(pyodide config get meson_cross_file) + +# Build in a persistent build directory +${UV_RUN_PREFIX} pyodide build \ + -Csetup-args=-Dallow-noblas=true \ + -Csetup-args=--cross-file="${MESON_CROSS_FILE}" \ + -Cinstall-args=--tags=runtime,python-runtime,devel \ + -Cbuild-dir="build" \ + --no-isolation --skip-dependency-check + +sed -i 's/numpy/numpy-tests/g' pyproject.toml + +${UV_RUN_PREFIX} pyodide build \ + -Csetup-args=-Dallow-noblas=true \ + -Csetup-args=--cross-file="${MESON_CROSS_FILE}" \ + -Cinstall-args=--tags=tests \ + -Cbuild-dir="build" \ + --no-isolation --skip-dependency-check + +echo "Successfully built wheels for numpy-${VERSION} and numpy-tests-${VERSION}." From abad178406cc9b276963c3299a380c932e99dafd Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 07:35:10 +0530 Subject: [PATCH 18/32] Use bash shell for [integration] tests --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 991dfd90..5b9223f9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,6 +154,7 @@ jobs: - name: Run the recipe integration tests (${{ matrix.task }}) if: matrix.task.name != 'test-integration-marker' + shell: bash env: PYODIDE_JOBS: ${{ steps.get-cores.outputs.CORES }} working-directory: integration_tests From d586dc49d0ebf411d5ea2fd07f7eed526da69151 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 07:45:56 +0530 Subject: [PATCH 19/32] Revert "Use bash shell for [integration] tests" This reverts commit abad178406cc9b276963c3299a380c932e99dafd. --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b9223f9..991dfd90 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,7 +154,6 @@ jobs: - name: Run the recipe integration tests (${{ matrix.task }}) if: matrix.task.name != 'test-integration-marker' - shell: bash env: PYODIDE_JOBS: ${{ steps.get-cores.outputs.CORES }} working-directory: integration_tests From b81e52538e13bcd4b7555d81478038730b6d7720 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 07:46:13 +0530 Subject: [PATCH 20/32] `source` doesn't work for some reason [integration] --- integration_tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/Makefile b/integration_tests/Makefile index f77e859c..05dd79b4 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -32,7 +32,7 @@ test-src-no-isolation: check @( \ set -e; \ python -m venv ./test_venv; \ - source ./test_venv/bin/activate; \ + . ./test_venv/bin/activate; \ pip install meson-python meson cython; \ pip install -e ../; \ ./src/numpy_no_isolation.sh; \ From 2c5ce4bab31c11e9dd2d4c989c68f2b8461a2c6c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 08:04:48 +0530 Subject: [PATCH 21/32] Miscellaneous edits --- integration_tests/src/numpy.sh | 2 +- integration_tests/src/numpy_no_isolation.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/integration_tests/src/numpy.sh b/integration_tests/src/numpy.sh index 465052b8..1abe6b44 100755 --- a/integration_tests/src/numpy.sh +++ b/integration_tests/src/numpy.sh @@ -10,4 +10,4 @@ tar -xf numpy-${VERSION}.tar.gz cd numpy-${VERSION} MESON_CROSS_FILE=$(pyodide config get meson_cross_file) -${UV_RUN_PREFIX} pyodide build -Csetup-args=-Dallow-noblas=true -Csetup-args=--cross-file="${MESON_CROSS_FILE}" "$@" +${UV_RUN_PREFIX} pyodide build -Csetup-args=-Dallow-noblas=true -Csetup-args=--cross-file="${MESON_CROSS_FILE}" diff --git a/integration_tests/src/numpy_no_isolation.sh b/integration_tests/src/numpy_no_isolation.sh index 8fc2b69b..82379d8c 100755 --- a/integration_tests/src/numpy_no_isolation.sh +++ b/integration_tests/src/numpy_no_isolation.sh @@ -1,6 +1,8 @@ #!/bin/bash -# The same as "numpy.sh", but without the isolation. +# The same as "numpy.sh", but without the isolation, and +# builds both NumPy and numpy-tests from a persistent +# build directory. set -e From f654818f8bbfe90be9a21c502364e857343afdcc Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 08:17:28 +0530 Subject: [PATCH 22/32] Add a CHANGELOG entry for #152 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d587b3c..b13c4d92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added basic support for uv. `uv tool install pyodide-cli --with pyodide-build`, or `uvx --from pyodide-cli --with pyodide-build pyodide --help`, or using `pyodide-build` in `uv`-managed virtual environments will now work. [#132](https://github.com/pyodide/pyodide-build/pull/132) +- Added support for building without a wheel without build isolation: `pyodide build` no accepts + the `--no-isolation`/`-n` and/or `--skip-dependency-check`/`-x` flags to customise the wheel + building behaviour, similar to `pypa/build`. + ### Changed - The Rust toolchain version has been updated to `nightly-2025-01-18`. From 1b879bdfd891b82036ecebb5389570cf25b65e73 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 31 Mar 2025 08:17:58 +0530 Subject: [PATCH 23/32] Trigger [integration] tests From 689c284e1dcf2b27bfa5513a6aaf42be528b425a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 1 Apr 2025 22:33:28 +0530 Subject: [PATCH 24/32] Trigger [integration] tests From 21562ecffd466901f201e291ebd0c2ec3a571648 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 1 Apr 2025 23:13:27 +0530 Subject: [PATCH 25/32] Try to fix geos recipe [integration] --- integration_tests/recipes/geos/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/recipes/geos/meta.yaml b/integration_tests/recipes/geos/meta.yaml index 77d763dc..187062d6 100644 --- a/integration_tests/recipes/geos/meta.yaml +++ b/integration_tests/recipes/geos/meta.yaml @@ -1,11 +1,11 @@ package: name: geos - version: 3.12.1 + version: 3.13.1 tag: - library source: - url: https://github.com/libgeos/geos/releases/download/3.12.1/geos-3.12.1.tar.bz2 - sha256: d6ea7e492224b51193e8244fe3ec17c4d44d0777f3c32ca4fb171140549a0d03 + url: https://github.com/libgeos/geos/releases/download/3.13.1/geos-3.13.1.tar.bz2 + sha256: df2c50503295f325e7c8d7b783aca8ba4773919cde984193850cf9e361dfd28c build: type: shared_library From 6088abac8fa777155f1f8dc606063e26271944d5 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 1 Apr 2025 23:26:25 +0530 Subject: [PATCH 26/32] `--fresh` is no longer needed [integration] --- pyodide_build/pywasmcross.py | 4 ---- pyodide_build/tests/test_pywasmcross.py | 1 - 2 files changed, 5 deletions(-) diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py index a766b15d..13b2ee64 100755 --- a/pyodide_build/pywasmcross.py +++ b/pyodide_build/pywasmcross.py @@ -521,10 +521,6 @@ def handle_command_generate_args( # noqa: C901 "emcmake", "cmake", *flags, - # Since we create a temporary directory and install compiler symlinks every time, - # CMakeCache.txt will contain invalid paths to the compiler when re-running, - # so we need to tell CMake to ignore the existing cache and build from scratch. - "--fresh", ] return line elif cmd == "meson": diff --git a/pyodide_build/tests/test_pywasmcross.py b/pyodide_build/tests/test_pywasmcross.py index 6b7381c2..f6bf9a90 100644 --- a/pyodide_build/tests/test_pywasmcross.py +++ b/pyodide_build/tests/test_pywasmcross.py @@ -257,7 +257,6 @@ def test_get_cmake_compiler_flags(): def test_handle_command_cmake(build_args): args = build_args - assert "--fresh" in handle_command_generate_args(["cmake", "./"], args) build_cmd = ["cmake", "--build", "." "--target", "target"] assert handle_command_generate_args(build_cmd, args) == build_cmd From 206a28e68c8b731827e7876048beb90f16179607 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 5 Apr 2025 15:51:34 +0530 Subject: [PATCH 27/32] Add type annotation for `_format_missing_dependencies` --- pyodide_build/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyodide_build/common.py b/pyodide_build/common.py index 65b11a69..f75d7b73 100644 --- a/pyodide_build/common.py +++ b/pyodide_build/common.py @@ -387,7 +387,7 @@ def _format_dep_chain(dep_chain: Sequence[str]) -> str: return " -> ".join(dep.partition(";")[0].strip() for dep in dep_chain) -def _format_missing_dependencies(missing) -> str: +def _format_missing_dependencies(missing: set[tuple[str, ...]]) -> str: return "".join( "\n\t" + dep for deps in missing From de93d2b62c6119ea245dbeeb674d0b0301f20bca Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 5 Apr 2025 16:10:23 +0530 Subject: [PATCH 28/32] Extract away directory building logic --- pyodide_build/out_of_tree/build.py | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/pyodide_build/out_of_tree/build.py b/pyodide_build/out_of_tree/build.py index ca6f2e4d..5016f089 100644 --- a/pyodide_build/out_of_tree/build.py +++ b/pyodide_build/out_of_tree/build.py @@ -9,6 +9,29 @@ from pyodide_build.spec import _BuildSpecExports +def prepare_build_dir(build_dir: Path) -> None: + # create a persistent build dir in the source dir + # don't track the build dir in git (helps if building in a git/mercurial repo) + build_dir.mkdir(exist_ok=True) + gitignore_path = build_dir / ".gitignore" + if not gitignore_path.exists(): + with gitignore_path.open("w") as f: + f.write( + dedent("""\ + # Created by pyodide-build + *""") + ) + hgignore_path = build_dir / ".hgignore" + if not hgignore_path.exists(): + with hgignore_path.open("w") as f: + f.write( + dedent("""\ + # Created by pyodide-build + syntax: glob + **/*""") + ) + + def run( srcdir: Path, outdir: Path, @@ -31,18 +54,8 @@ def run( env = os.environ.copy() env.update(build_env.get_build_environment_vars(get_pyodide_root())) - # create a persistent build dir in the source dir build_dir = srcdir / ".pyodide_build" - build_dir.mkdir(exist_ok=True) - # don't track the build dir (helps if building in a git repo) - gitignore_path = build_dir / ".gitignore" - if not gitignore_path.exists(): - with gitignore_path.open("w") as f: - f.write( - dedent("""\ - # Created by pyodide-build - *""") - ) + prepare_build_dir(build_dir) build_env_ctx = pypabuild.get_build_env( env=env, From 6ef79be4e11702e9139af8001b989859548ea2d8 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 5 Apr 2025 16:19:42 +0530 Subject: [PATCH 29/32] Extract away version control ignore even more --- pyodide_build/out_of_tree/build.py | 44 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/pyodide_build/out_of_tree/build.py b/pyodide_build/out_of_tree/build.py index 5016f089..8b1935e4 100644 --- a/pyodide_build/out_of_tree/build.py +++ b/pyodide_build/out_of_tree/build.py @@ -9,27 +9,31 @@ from pyodide_build.spec import _BuildSpecExports -def prepare_build_dir(build_dir: Path) -> None: +def _create_ignore_files(directory: Path) -> None: + directory.joinpath(".gitignore").write_text( + dedent("""\ + # Created by pyodide-build + * + """), + encoding="utf-8", + ) + + directory.joinpath(".hgignore").write_text( + dedent("""\ + # Created by pyodide-build + syntax: glob + **/* + """), + encoding="utf-8", + ) + + +def _prepare_build_dir(build_dir: Path) -> None: # create a persistent build dir in the source dir - # don't track the build dir in git (helps if building in a git/mercurial repo) build_dir.mkdir(exist_ok=True) - gitignore_path = build_dir / ".gitignore" - if not gitignore_path.exists(): - with gitignore_path.open("w") as f: - f.write( - dedent("""\ - # Created by pyodide-build - *""") - ) - hgignore_path = build_dir / ".hgignore" - if not hgignore_path.exists(): - with hgignore_path.open("w") as f: - f.write( - dedent("""\ - # Created by pyodide-build - syntax: glob - **/*""") - ) + # don't track the build dir in version control, + # helps if building in a git/mercurial repo + _create_ignore_files(build_dir) def run( @@ -55,7 +59,7 @@ def run( env.update(build_env.get_build_environment_vars(get_pyodide_root())) build_dir = srcdir / ".pyodide_build" - prepare_build_dir(build_dir) + _prepare_build_dir(build_dir) build_env_ctx = pypabuild.get_build_env( env=env, From 920aec7a56a778df4bf25961ebff9056f1655803 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 5 Apr 2025 16:20:03 +0530 Subject: [PATCH 30/32] `no_isolation` is no longer used --- pyodide_build/pypabuild.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index 77eff0de..e3605ed6 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -286,7 +286,8 @@ def make_command_wrapper_symlinks(symlink_dir: Path) -> dict[str, str]: @contextmanager def _create_symlink_dir( - env: dict[str, str], build_dir: Path, no_isolation: bool = False + env: dict[str, str], + build_dir: Path, ): # Leave the symlinks in the build directory. This helps with reproducing. symlink_dir = build_dir / "pywasmcross_symlinks" @@ -325,7 +326,7 @@ def get_build_env( args["exports"] = exports env = env.copy() - with _create_symlink_dir(env, build_dir, no_isolation) as symlink_dir: + with _create_symlink_dir(env, build_dir) as symlink_dir: env.update(make_command_wrapper_symlinks(symlink_dir)) sysconfig_dir = Path(get_build_flag("TARGETINSTALLDIR")) / "sysconfigdata" args["PYTHONPATH"] = sys.path + [str(symlink_dir), str(sysconfig_dir)] From ba3e8ba51f4df3fa83578c27e2135bbcac2eb74c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 5 Apr 2025 17:07:42 +0530 Subject: [PATCH 31/32] Add TODO for `_create_symlink_dir` context manager --- pyodide_build/pypabuild.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index e3605ed6..3a639722 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -284,6 +284,7 @@ def make_command_wrapper_symlinks(symlink_dir: Path) -> dict[str, str]: return env +# TODO: a context manager is no longer needed here @contextmanager def _create_symlink_dir( env: dict[str, str], From db53bdbe2c967edb64e1697366b859de76996839 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 5 Apr 2025 17:25:00 +0530 Subject: [PATCH 32/32] Trigger [integration] tests