From 8598beb25729567f50c00c8749808f9ad46bc7c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sun, 14 Sep 2025 20:51:49 +0200 Subject: [PATCH 1/3] cffi: Move CFFI logic from global scope into `get_ffi()` function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the CFFI-related logic from the global scope of `make_cffi.py` file to a `get_ffi()` function. This makes it possible to import the file without executing the code immediately, and it will make it possible to parametrize the invocation using explicit function parameters. Along with the change, I've renamed the variables to lowercase since they are no longer global. Signed-off-by: Michał Górny --- make_cffi.py | 137 ++++++++++++++++++++++++++------------------------- setup.py | 2 +- 2 files changed, 71 insertions(+), 68 deletions(-) diff --git a/make_cffi.py b/make_cffi.py index 9b5fc0f5..2227fe3b 100644 --- a/make_cffi.py +++ b/make_cffi.py @@ -16,22 +16,6 @@ import cffi import packaging.tags -HERE = os.path.abspath(os.path.dirname(__file__)) - -SOURCES = [ - "zstd/zstd.c", -] - -# Headers whose preprocessed output will be fed into cdef(). -HEADERS = [ - os.path.join(HERE, "zstd", p) - for p in ("zstd_errors.h", "zstd.h", "zdict.h") -] - -INCLUDE_DIRS = [ - os.path.join(HERE, "zstd"), -] - # cffi can't parse some of the primitives in zstd.h. So we invoke the # preprocessor and feed its output into cffi. compiler = distutils.ccompiler.new_compiler() @@ -168,21 +152,38 @@ def normalize_output(output): return b"\n".join(lines) -# musl 1.1 doesn't define qsort_r. We need to force using the C90 -# variant. -define_macros = [] -for tag in packaging.tags.platform_tags(): - if tag.startswith("musllinux_1_1_"): - define_macros.append(("ZDICT_QSORT", "ZDICT_QSORT_C90")) +def get_ffi(system_zstd = False): + here = os.path.abspath(os.path.dirname(__file__)) + + zstd_sources = [ + "zstd/zstd.c", + ] + # Headers whose preprocessed output will be fed into cdef(). + headers = [ + os.path.join(here, "zstd", p) + for p in ("zstd_errors.h", "zstd.h", "zdict.h") + ] -ffi = cffi.FFI() -# *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning -# when cffi uses the function. Since we statically link against zstd, even -# if we use the deprecated functions it shouldn't be a huge problem. -ffi.set_source( - "zstandard._cffi", - """ + include_dirs = [ + os.path.join(here, "zstd"), + ] + + # musl 1.1 doesn't define qsort_r. We need to force using the C90 + # variant. + define_macros = [] + for tag in packaging.tags.platform_tags(): + if tag.startswith("musllinux_1_1_"): + define_macros.append(("ZDICT_QSORT", "ZDICT_QSORT_C90")) + + + ffi = cffi.FFI() + # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning + # when cffi uses the function. Since we statically link against zstd, even + # if we use the deprecated functions it shouldn't be a huge problem. + ffi.set_source( + "zstandard._cffi", + """ #define ZSTD_STATIC_LINKING_ONLY #define ZSTD_DISABLE_DEPRECATE_WARNINGS #include @@ -191,48 +192,50 @@ def normalize_output(output): #define ZDICT_DISABLE_DEPRECATE_WARNINGS #include """, - sources=SOURCES, - include_dirs=INCLUDE_DIRS, - define_macros=define_macros, -) - -DEFINE = re.compile(rb"^#define\s+([a-zA-Z0-9_]+)\s+(\S+)") - -sources = [] - -# Feed normalized preprocessor output for headers into the cdef parser. -for header in HEADERS: - preprocessed = preprocess(header) - sources.append(normalize_output(preprocessed)) - - # #define's are effectively erased as part of going through preprocessor. - # So perform a manual pass to re-add those to the cdef source. - with open(header, "rb") as fh: - for line in fh: - line = line.strip() - m = DEFINE.match(line) - if not m: - continue + sources=zstd_sources, + include_dirs=include_dirs, + define_macros=define_macros, + ) - if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY": - continue + define = re.compile(rb"^#define\s+([a-zA-Z0-9_]+)\s+(\S+)") - # The parser doesn't like some constants with complex values. - if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"): - continue + sources = [] - # These defines create aliases from old (camelCase) type names - # to the new PascalCase names, which breaks CFFI. - if m.group(1).lower() == m.group(2).lower(): - continue + # Feed normalized preprocessor output for headers into the cdef parser. + for header in headers: + preprocessed = preprocess(header) + sources.append(normalize_output(preprocessed)) + + # #define's are effectively erased as part of going through preprocessor. + # So perform a manual pass to re-add those to the cdef source. + with open(header, "rb") as fh: + for line in fh: + line = line.strip() + m = define.match(line) + if not m: + continue + + if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY": + continue + + # The parser doesn't like some constants with complex values. + if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"): + continue + + # These defines create aliases from old (camelCase) type names + # to the new PascalCase names, which breaks CFFI. + if m.group(1).lower() == m.group(2).lower(): + continue + + # The ... is magic syntax by the cdef parser to resolve the + # value at compile time. + sources.append(b"#define " + m.group(1) + b" ...") - # The ... is magic syntax by the cdef parser to resolve the - # value at compile time. - sources.append(b"#define " + m.group(1) + b" ...") + cdeflines = b"\n".join(sources).splitlines() + cdeflines = [line for line in cdeflines if line.strip()] + ffi.cdef(b"\n".join(cdeflines).decode("latin1")) + return ffi -cdeflines = b"\n".join(sources).splitlines() -cdeflines = [line for line in cdeflines if line.strip()] -ffi.cdef(b"\n".join(cdeflines).decode("latin1")) if __name__ == "__main__": - ffi.compile() + get_ffi().compile() diff --git a/setup.py b/setup.py index 5bc5558f..ab64481d 100755 --- a/setup.py +++ b/setup.py @@ -138,7 +138,7 @@ if CFFI_BACKEND and cffi: import make_cffi - extensions.append(make_cffi.ffi.distutils_extension()) + extensions.append(make_cffi.get_ffi().distutils_extension()) version = None From 5fd00c314a7d3ef09e26ed91bf552662a8e61a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 24 Jan 2025 15:54:02 +0100 Subject: [PATCH 2/3] cffi: Support --system-zstd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support using the system zstd library with the CFFI backend. Use the GCC / Clang preprocessor output to find the paths to header files for preprocessing. Link to the system library, matching the behavior for the C backend. Signed-off-by: Michał Górny --- make_cffi.py | 49 +++++++++++++++++++++++++++++++++++++------------ setup.py | 2 +- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/make_cffi.py b/make_cffi.py index 2227fe3b..bed91617 100644 --- a/make_cffi.py +++ b/make_cffi.py @@ -153,21 +153,45 @@ def normalize_output(output): def get_ffi(system_zstd = False): - here = os.path.abspath(os.path.dirname(__file__)) + zstd_sources = [] + include_dirs = [] + libraries = [] - zstd_sources = [ - "zstd/zstd.c", - ] + if not system_zstd: + here = os.path.abspath(os.path.dirname(__file__)) - # Headers whose preprocessed output will be fed into cdef(). - headers = [ - os.path.join(here, "zstd", p) - for p in ("zstd_errors.h", "zstd.h", "zdict.h") - ] + zstd_sources += [ + "zstd/zstd.c", + ] + + # Headers whose preprocessed output will be fed into cdef(). + headers = [ + os.path.join(here, "zstd", p) + for p in ("zstd_errors.h", "zstd.h", "zdict.h") + ] - include_dirs = [ - os.path.join(here, "zstd"), - ] + include_dirs += [ + os.path.join(here, "zstd"), + ] + else: + libraries += ["zstd"] + + # Locate headers using the preprocessor. + include_re = re.compile(r'^# \d+ "([^"]+/(?:zstd_errors|zstd|zdict)\.h)"') + with tempfile.TemporaryDirectory() as temp_dir: + with open(os.path.join(temp_dir, "input.h"), "w") as f: + f.write(""" +#include +#include +#include +""") + compiler.preprocess(os.path.join(temp_dir, "input.h"), + os.path.join(temp_dir, "output.h")) + with open(os.path.join(temp_dir, "output.h"), "r") as f: + headers = list({ + m.group(1) for m in map(include_re.match, f) + if m is not None + }) # musl 1.1 doesn't define qsort_r. We need to force using the C90 # variant. @@ -195,6 +219,7 @@ def get_ffi(system_zstd = False): sources=zstd_sources, include_dirs=include_dirs, define_macros=define_macros, + libraries=libraries, ) define = re.compile(rb"^#define\s+([a-zA-Z0-9_]+)\s+(\S+)") diff --git a/setup.py b/setup.py index ab64481d..e6ea3e4b 100755 --- a/setup.py +++ b/setup.py @@ -138,7 +138,7 @@ if CFFI_BACKEND and cffi: import make_cffi - extensions.append(make_cffi.get_ffi().distutils_extension()) + extensions.append(make_cffi.get_ffi(system_zstd=SYSTEM_ZSTD).distutils_extension()) version = None From 6158c6fd9d5860768cd946ddc6bf8dbc272ac4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 16 Sep 2025 14:16:32 +0200 Subject: [PATCH 3/3] Use a separate switch for system zstd with cffi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Given that the CFFI code is much more fragile to library changes than the C extension code, use a separate `--system-zstd-cffi` switch to enable it. This should resolve the test failure on macOS. Signed-off-by: Michał Górny --- .github/workflows/external-zstd.yml | 2 +- setup.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/external-zstd.yml b/.github/workflows/external-zstd.yml index 6591b1cb..b988a582 100644 --- a/.github/workflows/external-zstd.yml +++ b/.github/workflows/external-zstd.yml @@ -29,7 +29,7 @@ jobs: - name: Build run: | - python -m pip install --config-settings=--global-option=--system-zstd . + python -m pip install --config-settings=--global-option=--system-zstd --config-settings=--global-option=--system-zstd-cffi . macOS: runs-on: 'macos-13' diff --git a/setup.py b/setup.py index e6ea3e4b..8df4b686 100755 --- a/setup.py +++ b/setup.py @@ -73,6 +73,7 @@ SUPPORT_LEGACY = False SYSTEM_ZSTD = False +SYSTEM_ZSTD_CFFI = False WARNINGS_AS_ERRORS = False C_BACKEND = True CFFI_BACKEND = True @@ -103,6 +104,10 @@ SYSTEM_ZSTD = True sys.argv.remove("--system-zstd") +if "--system-zstd-cffi" in sys.argv: + SYSTEM_ZSTD_CFFI = True + sys.argv.remove("--system-zstd-cffi") + if "--warnings-as-errors" in sys.argv: WARNINGS_AS_ERRORS = True sys.argv.remove("--warning-as-errors") @@ -138,7 +143,7 @@ if CFFI_BACKEND and cffi: import make_cffi - extensions.append(make_cffi.get_ffi(system_zstd=SYSTEM_ZSTD).distutils_extension()) + extensions.append(make_cffi.get_ffi(system_zstd=SYSTEM_ZSTD_CFFI).distutils_extension()) version = None