From 69aa386a4a3bc03b1161df51d098cd6cd7014796 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 3 Feb 2026 15:03:32 +0900 Subject: [PATCH] feat: adjust make.py for ad-hoc builds --- adbc_drivers_dev/make.py | 211 +++++++++++++++++---------- adbc_drivers_dev/templates/test.yaml | 1 - 2 files changed, 130 insertions(+), 82 deletions(-) diff --git a/adbc_drivers_dev/make.py b/adbc_drivers_dev/make.py index 3740961..cbb78eb 100644 --- a/adbc_drivers_dev/make.py +++ b/adbc_drivers_dev/make.py @@ -32,10 +32,13 @@ match platform.system(): case "Darwin": EXT = "dylib" + PLATFORM = "macos" case "Linux": EXT = "so" + PLATFORM = "linux" case "Windows": EXT = "dll" + PLATFORM = "windows" case _: raise RuntimeError(f"Unsupported platform: {platform.system()}") @@ -43,6 +46,7 @@ DOIT_CONFIG = { "default_tasks": ["build"], } +SMUGGLE_VARS = {"CGO_CFLAGS", "CGO_LDFLAGS", "PROTOC"} def to_bool(value: str | bool) -> bool: @@ -140,9 +144,6 @@ def detect_version( *, strict: bool = False, ) -> str: - if not any((driver_root / name).is_file() for name in ("Cargo.toml", "go.mod")): - raise ValueError(f"{driver_root} does not contain a Cargo.toml or go.mod") - repo_root = driver_root while not (repo_root / ".git").is_dir(): if repo_root.parent == repo_root: @@ -211,6 +212,58 @@ def get_var(name: str, default: str) -> str: return value +def maybe_build_docker( + *, + repo_root: Path, + driver_root: Path, + env: dict[str, str], + args: list[str], + ci: bool, +) -> None: + if not ci or platform.system() != "Linux": + check_call(args, cwd=driver_root, env=env) + return + + env = env.copy() + env["SOURCE_ROOT"] = str(repo_root) + env["ARCH"] = architecture() + + volumes = get_var("ADDITIONAL_VOLUMES", "") + if volumes: + volumes = volumes.split(",") + + # Some env vars need to be explicitly propagated into Docker + smuggle_env = "" + for var in SMUGGLE_VARS: + if var in env: + smuggle_env += f'{var}="{shlex.quote(env[var])}" ' + elif var in os.environ: + smuggle_env += f'{var}="{shlex.quote(os.environ[var])}" ' + + command = [ + "docker", + "compose", + "run", + "--rm", + "--user", + str(os.getuid()), + ] + + for volume in volumes: + command.extend(["-v", volume]) + + command.extend( + [ + "manylinux-rust", + "--", + "bash", + "-c", + f"cd /source/{driver_root.relative_to(repo_root)} && env {smuggle_env} {' '.join(args)}", + ] + ) + check_call(command, cwd=Path(__file__).parent, env=env) + + def build_go( repo_root: Path, driver_root: Path, @@ -249,9 +302,7 @@ def build_go( info("Building", target, "version", version) env = {} - - smuggle_vars = ("CGO_CFLAGS", "CGO_LDFLAGS") - for var in smuggle_vars: + for var in SMUGGLE_VARS: if var in os.environ: env[var] = os.environ[var] @@ -260,33 +311,29 @@ def build_go( append_flags(env, "CGO_LDFLAGS", "-mmacosx-version-min=11.0") if ci and platform.system() == "Linux": - env["SOURCE_ROOT"] = str(repo_root) - env["ARCH"] = architecture() - check_call(["go", "mod", "vendor"], cwd=driver_root) - - smuggle_env = "" - for var in smuggle_vars: - if var in env: - smuggle_env += f'{var}="{shlex.quote(env[var])}" ' - ldflags += ( " -linkmode external -extldflags=-Wl,--version-script=/only-export-adbc.ld" ) - command = [ - "docker", - "compose", - "run", - "--rm", - "--user", - str(os.getuid()), - "manylinux", - "--", - "bash", - "-c", - f'cd /source/{driver_root.relative_to(repo_root)} && env {smuggle_env} go build -buildmode=c-shared {tags} -o /source/build/{target} -ldflags "{ldflags}" ./pkg', - ] - check_call(command, cwd=Path(__file__).parent, env=env) + + # Command differs under Docker so don't invoke this otherwise + maybe_build_docker( + repo_root=repo_root, + driver_root=driver_root, + env=env, + args=[ + "go", + "build", + "-buildmode=c-shared", + tags, + "-o", + f"/source/build/{target}", + "-ldflags", + ldflags, + "./pkg", + ], + ci=ci, + ) else: check_call( [ @@ -344,60 +391,17 @@ def build_rust( info("Building", target, "version", version, "features", features) env = {} - # Some env vars need to be explicitly propagated into Docker - smuggle_vars = {"PROTOC"} - if platform.system() == "Darwin": # https://doc.rust-lang.org/nightly/rustc/platform-support/apple-darwin.html#os-version env["MACOSX_DEPLOYMENT_TARGET"] = "11.0" - if ci and platform.system() == "Linux": - env["SOURCE_ROOT"] = str(repo_root) - env["ARCH"] = architecture() - - volumes = get_var("ADDITIONAL_VOLUMES", "") - if volumes: - volumes = volumes.split(",") - - smuggle_env = "" - for var in smuggle_vars: - if var in env: - smuggle_env += f'{var}="{shlex.quote(env[var])}" ' - elif var in os.environ: - smuggle_env += f'{var}="{shlex.quote(os.environ[var])}" ' - - command = [ - "docker", - "compose", - "run", - "--rm", - "--user", - str(os.getuid()), - ] - - for volume in volumes: - command.extend(["-v", volume]) - - command.extend( - [ - "manylinux-rust", - "--", - "bash", - "-c", - f"cd /source/{driver_root.relative_to(repo_root)} && env {smuggle_env} cargo build {' '.join(args)}", - ] - ) - check_call(command, cwd=Path(__file__).parent, env=env) - else: - check_call( - [ - "cargo", - "build", - *args, - ], - cwd=driver_root, - env=env, - ) + maybe_build_docker( + repo_root=repo_root, + driver_root=driver_root, + env=env, + args=["cargo", "build", *args], + ci=ci, + ) lib = driver_root / "target" if debug: @@ -415,6 +419,45 @@ def build_rust( output.chmod(0o755) +def build_custom( + repo_root: Path, + driver_root: Path, + driver: str, + target: str, + *, + ci: bool = False, +) -> None: + version = detect_version(driver_root) + (repo_root / "build").mkdir(exist_ok=True) + + debug = to_bool(get_var("DEBUG", "False")) + + args = [] + if debug: + args.append("release") + else: + args.append("test") + args.append(PLATFORM) + args.append(architecture()) + + info("Building", target, "version", version) + + env = {} + if platform.system() == "Darwin": + env["MACOSX_DEPLOYMENT_TARGET"] = "11.0" + + maybe_build_docker( + repo_root=repo_root, + driver_root=driver_root, + env=env, + args=["./ci/scripts/build.sh", *args], + ci=ci, + ) + + output = (repo_root / "build" / target).resolve() + output.chmod(0o755) + + def check_linux(binary: Path) -> None: symbols = check_output( [ @@ -492,14 +535,16 @@ def task_build(): if not driver: raise ValueError("Must specify DRIVER=driver") - ci = get_var("CI", False) + ci = to_bool(get_var("CI", False)) lang = get_var("IMPL_LANG", "go").strip().lower() repo_root = Path(".").resolve().absolute() driver_root = Path(driver) if driver_root.is_dir(): driver_root = driver_root.resolve() - elif Path("./go.mod").is_file() or Path("./Cargo.toml").is_file(): + elif ( + Path("./go.mod").is_file() or Path("./Cargo.toml").is_file() or lang == "custom" + ): driver_root = Path(".").resolve() # Compute dependencies @@ -522,6 +567,10 @@ def task_build(): actions = [ lambda: build_rust(repo_root, driver_root, driver, target, ci=ci), ] + elif lang == "custom": + actions = [ + lambda: build_custom(repo_root, driver_root, driver, target, ci=ci), + ] else: raise ValueError(f"Unsupported LANG={lang}") diff --git a/adbc_drivers_dev/templates/test.yaml b/adbc_drivers_dev/templates/test.yaml index 9dd7ba6..1c2fd9f 100644 --- a/adbc_drivers_dev/templates/test.yaml +++ b/adbc_drivers_dev/templates/test.yaml @@ -377,7 +377,6 @@ jobs: fi set +a pixi run adbc-make build DEBUG=true VERBOSE=true DRIVER=<{driver}> IMPL_LANG=<{lang}> <{' '.join(lang_config.build.additional_make_args) }> - - name: Start Test Dependencies # Can't use Docker on macOS AArch64 runners, and Windows containers # work but often the container doesn't support Windows