From 9fa8340a74097bbed3db03b9aa4739bfdf20e403 Mon Sep 17 00:00:00 2001 From: roblabla Date: Tue, 30 Jul 2024 11:01:24 +0200 Subject: [PATCH 01/12] Fix unknown OS error when building for windows To successfully build jemalloc for msvc-windows, we need to pretend we're doing a mingw build. The configure script will then check if the compiler is actually an MSVC compiler (by checking if it defines _MSC_VER), and if so switch to an MSVC build (which enables a bunch of compatibility hacks, such as removing the need for pthread). Signed-off-by: roblabla --- jemalloc-sys/build.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jemalloc-sys/build.rs b/jemalloc-sys/build.rs index b91d3e3d6..37b9ee7c3 100644 --- a/jemalloc-sys/build.rs +++ b/jemalloc-sys/build.rs @@ -395,8 +395,13 @@ fn execute(cmd: &mut Command, on_fail: impl FnOnce()) { fn gnu_target(target: &str) -> String { match target { - "i686-pc-windows-msvc" => "i686-pc-win32".to_string(), - "x86_64-pc-windows-msvc" => "x86_64-pc-win32".to_string(), + // We need to lie and pretend to be mingw32 for MSVC builds. Autoconf + // will then check if the compiler is MSVC, and if so, switch to an + // MSVC-based build. + // + // See https://github.com/jemalloc/jemalloc/blob/5.3.0/configure.ac#L750 + "i686-pc-windows-msvc" => "i686-pc-mingw32".to_string(), + "x86_64-pc-windows-msvc" => "x86_64-pc-mingw32".to_string(), "i686-pc-windows-gnu" => "i686-w64-mingw32".to_string(), "x86_64-pc-windows-gnu" => "x86_64-w64-mingw32".to_string(), "armv7-linux-androideabi" => "arm-linux-androideabi".to_string(), From 99ba03b9d9274275f6eceb48a92530a786292052 Mon Sep 17 00:00:00 2001 From: roblabla Date: Tue, 30 Jul 2024 11:03:29 +0200 Subject: [PATCH 02/12] Fix generated lib name on windows targets On windows, the generated library is called jemalloc_s instead of jemalloc_pic. Signed-off-by: roblabla --- jemalloc-sys/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jemalloc-sys/build.rs b/jemalloc-sys/build.rs index 37b9ee7c3..20f63f82c 100644 --- a/jemalloc-sys/build.rs +++ b/jemalloc-sys/build.rs @@ -339,7 +339,7 @@ fn main() { // intrinsics that are libgcc specific (e.g. those intrinsics aren't present in // libcompiler-rt), so link that in to get that support. if target.contains("windows") { - println!("cargo:rustc-link-lib=static=jemalloc"); + println!("cargo:rustc-link-lib=static=jemalloc_s"); } else { println!("cargo:rustc-link-lib=static=jemalloc_pic"); } From 8766f1167657bb3ea4183b9b3083158c2f036e52 Mon Sep 17 00:00:00 2001 From: roblabla Date: Tue, 30 Jul 2024 11:04:12 +0200 Subject: [PATCH 03/12] Add support for win7 targets Signed-off-by: roblabla --- jemalloc-sys/build.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jemalloc-sys/build.rs b/jemalloc-sys/build.rs index 20f63f82c..45933d0bb 100644 --- a/jemalloc-sys/build.rs +++ b/jemalloc-sys/build.rs @@ -401,7 +401,9 @@ fn gnu_target(target: &str) -> String { // // See https://github.com/jemalloc/jemalloc/blob/5.3.0/configure.ac#L750 "i686-pc-windows-msvc" => "i686-pc-mingw32".to_string(), + "i686-win7-windows-msvc" => "i686-pc-mingw32".to_string(), "x86_64-pc-windows-msvc" => "x86_64-pc-mingw32".to_string(), + "x86_64-win7-windows-msvc" => "x86_64-pc-mingw32".to_string(), "i686-pc-windows-gnu" => "i686-w64-mingw32".to_string(), "x86_64-pc-windows-gnu" => "x86_64-w64-mingw32".to_string(), "armv7-linux-androideabi" => "arm-linux-androideabi".to_string(), From 5efb1303e4f5ce9870536134859f19fca29d0881 Mon Sep 17 00:00:00 2001 From: roblabla Date: Fri, 2 Aug 2024 15:57:06 +0200 Subject: [PATCH 04/12] Add windows targets to NO_UNPREFIXED_MALLOC_TARGET It turns out, jemalloc doesn't support having unprefixed malloc on these targets. When --with-jemalloc-prefix is not specified, jemalloc will implicitly add a `je_` prefix regardless. Signed-off-by: roblabla --- jemalloc-sys/src/env.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jemalloc-sys/src/env.rs b/jemalloc-sys/src/env.rs index 2053ad306..79cb55318 100644 --- a/jemalloc-sys/src/env.rs +++ b/jemalloc-sys/src/env.rs @@ -18,7 +18,12 @@ pub static NO_BG_THREAD_TARGETS: &[&str] = &["musl"]; // “it was found that the `realpath` function in libc would allocate with libc malloc // (not jemalloc malloc), and then the standard library would free with jemalloc free, // causing a segfault.” +// // https://github.com/rust-lang/rust/commit/e3b414d8612314e74e2b0ebde1ed5c6997d28e8d // https://github.com/rust-lang/rust/commit/9f3de647326fbe50e0e283b9018ab7c41abccde3 // https://github.com/rust-lang/rust/commit/ed015456a114ae907a36af80c06f81ea93182a24 -pub static NO_UNPREFIXED_MALLOC_TARGETS: &[&str] = &["android", "dragonfly", "darwin"]; +// +// Furthermore, macos (using macho abi) and windows (using pecoff abi) don't +// support unprefixed malloc at all: +// https://github.com/jemalloc/jemalloc/blob/8dc97b11089be6d58a52009ea3da610bf90331d3/configure.ac#L1109 +pub static NO_UNPREFIXED_MALLOC_TARGETS: &[&str] = &["android", "dragonfly", "darwin", "windows"]; From 73dc9305f3d56c73e5205dfa91b91ffbe3c296e8 Mon Sep 17 00:00:00 2001 From: roblabla Date: Sat, 3 Aug 2024 11:28:59 +0200 Subject: [PATCH 05/12] Add windows CI Signed-off-by: roblabla --- .github/workflows/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 721e5d089..81d4191bb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,6 +16,11 @@ jobs: strategy: matrix: include: + - name: x86_64-pc-windows-msvc + target: x86_64-pc-windows-msvc + nobgt: 0 + no_tests: 0 + tag: windows-latest - name: x86_64-apple-darwin target: x86_64-apple-darwin nobgt: 0 @@ -59,6 +64,7 @@ jobs: with: submodules: true - name: install + shell: bash run: | if [[ "${{ matrix.rust }}" == "nightly" ]]; then rustup default nightly From 77face5b7963531b0d409503dd7cd9a27c1af323 Mon Sep 17 00:00:00 2001 From: roblabla Date: Sat, 3 Aug 2024 21:04:47 +0200 Subject: [PATCH 06/12] Set cc-rs env vars when invoking MSVC CC When invoking MSVC CC, a handful of environment variables must be set for the compiler to find its supporting files (such as system includes, libraries, etc...). When invoking it through cc-rs, these environment variables are automatically detected and set. However, we currently don't set those environment variables when running `./configure` or `make`, leading to those commands failing due to the compiler not having access to any of the system include/libraries. To fix this, we ask cc-rs to give us the environment variables to set, and set them whenever running `configure` and `make`. Signed-off-by: roblabla --- jemalloc-sys/build.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/jemalloc-sys/build.rs b/jemalloc-sys/build.rs index 45933d0bb..9034e1817 100644 --- a/jemalloc-sys/build.rs +++ b/jemalloc-sys/build.rs @@ -198,6 +198,13 @@ fn main() { .arg("--enable-doc=no") .arg("--enable-shared=no"); + for (env_key, env_value) in compiler.env() { + // MSVC compilers require a handful of environment variables to be set + // to work properly, so they can find their built-in include folders and + // default library path. + cmd.env(env_key, env_value); + } + if target.contains("ios") { // newer iOS deviced have 16kb page sizes: // closed: https://github.com/gnzlbg/jemallocator/issues/68 @@ -302,10 +309,19 @@ fn main() { // Make: let make = make_cmd(&host); - run(Command::new(make) + let mut make_cmd = Command::new(make); + make_cmd .current_dir(&build_dir) .arg("-j") - .arg(num_jobs.clone())); + .arg(num_jobs.clone()); + + for (env_key, env_value) in compiler.env() { + // MSVC compilers require a handful of environment variables to be set + // to work properly, so they can find their built-in include folders and + // default library path. + make_cmd.env(env_key, env_value); + } + run(&mut make_cmd); // Skip watching this environment variables to avoid rebuild in CI. if env::var("JEMALLOC_SYS_RUN_JEMALLOC_TESTS").is_ok() { From 9c78a6ab1320fa82d918ae7ee9b8c5975648ceb4 Mon Sep 17 00:00:00 2001 From: roblabla Date: Sat, 3 Aug 2024 11:29:08 +0200 Subject: [PATCH 07/12] Attempt supporting spaces in CC/SHELL path Autoconf does not support spaces in the compiler path. This is not usually a problem, as on most platforms, the compiler will reside in a path without spaces (such as /usr/bin or /opt). However, on windows, the MSVC compiler is usually found under C:\Program Files, which causes all hell to break loose. Similarly, the jemalloc Makefile also assumes the shell (stored in the SHELL variable) is in a folder without spaces, and is used unquoted all throughout the makefile, causing issues if the shell is in C:\Program Files, such as when using Git Bash (which installs in C:\Program Files\Git\bin\bash). Thankfully, windows has something called "Short Paths", which we can use to turn C:\Program Files into C:\PROGRA~2, eliminating the path. If the build script detects the compiler path has a space, and is running on windows, it will attempt to turn the path into a short path and run with this. Signed-off-by: roblabla --- jemalloc-sys/build.rs | 175 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 155 insertions(+), 20 deletions(-) diff --git a/jemalloc-sys/build.rs b/jemalloc-sys/build.rs index 9034e1817..7f954a25d 100644 --- a/jemalloc-sys/build.rs +++ b/jemalloc-sys/build.rs @@ -14,8 +14,11 @@ use std::{ fs, io, path::{Path, PathBuf}, process::Command, + sync::OnceLock, }; +use cc::Tool; + include!("src/env.rs"); macro_rules! info { @@ -73,6 +76,116 @@ fn expect_env(name: &str) -> String { env::var(name).unwrap_or_else(|_| panic!("{} was not set", name)) } +/// Attempts to turn the given path into a short path. Returns none in case of +/// error. +fn to_short_path(orig_path: &Path) -> Option { + let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR was not set")); + let script_path = out_dir.join("long_to_short.bat"); + std::fs::write(&script_path, "@echo %~s1").unwrap(); + let path = match Command::new(script_path).arg(orig_path).output() { + Ok(v) if v.status.success() => v.stdout, + Ok(v) => { + warning!( + "Failed to expand {} to a short path, the command exited with status {}\nstdout: {}\nstderr: {}", + orig_path.display(), + v.status, + String::from_utf8_lossy(&v.stdout), + String::from_utf8_lossy(&v.stderr), + ); + return None; + } + Err(err) => { + warning!( + "Failed to expand {} to a short path: {}", + orig_path.display(), + err + ); + return None; + } + }; + + // Note: This will only work if the path is valid UTF8. In practice, this + // will always be the case (in fact, we're mostly interested in turning + // "C:\Program Files" into C:\PROG~1). + // + // Should we ever need non-utf8 support here, we'd need to replace the above + // script with a call to GetShortPathNameW, transforming orig_path into + // utf16 through OsStr::encode_wide and the resulting path back into an + // OsString through OsString::from_wide. + let path = match String::from_utf8(path) { + Ok(v) => v, + Err(_err) => { + warning!("Short path of {} is not valid utf-8", orig_path.display()); + return None; + } + }; + + Some(PathBuf::from(path.trim())) +} + +fn find_shell_arg(host: &str, build_dir: &Path) -> Option { + if !cfg!(windows) { + return None; + } + + let make = make_cmd_program(host); + + // Override SHELL to use short paths if it's set to a path using spaces. + // This is often the case if using git bash as a shell, since it will be + // found in C:\Program Files\Git\bin\bash.exe. Github Actions does that. + // + // First, we have to find the SHELL variable. This is often hardcoded + // into the make binary, so we have to use `make -p` to find its value. + let make_vars = Command::new(make) + .current_dir(build_dir) + .arg("-p") + // this stinks. + .arg("-fNUL") + .output(); + match make_vars { + Ok(make_vars_out) + if make_vars_out.status.success() || make_vars_out.status.code() == Some(2) => + { + let out = String::from_utf8_lossy(&make_vars_out.stdout); + let mut found_shell = false; + for line in out.lines() { + if line.starts_with("SHELL ") { + found_shell = true; + // Once we've found the value, we must turn it into a + // short_path if it contains a space. + if let Some(shell) = line.split("= ").nth(1) { + let shell = shell.trim(); + if shell.contains(' ') { + if let Some(short_path) = to_short_path(Path::new(shell)) { + // And finally, we override it using SHELL=X arg + // syntax of makefiles. + return Some(format!("SHELL={}", short_path.display())); + } + } + } + } + } + if !found_shell { + warning!("Did not find SHELL value in make database.",); + } + None + } + Ok(make_vars_out) => { + warning!( + "Failed to run {} -p -fNUL, exited with status {}\nStderr: {}", + make, + make_vars_out.status, + String::from_utf8_lossy(&make_vars_out.stderr) + ); + None + } + Err(err) => { + warning!("Failed to print builtin make values: {:?}", err); + None + } + } +} + // TODO: split main functions and remove following allow. #[allow(clippy::cognitive_complexity)] fn main() { @@ -154,7 +267,19 @@ fn main() { .map(|s| s.to_str().unwrap()) .collect::>() .join(" "); - info!("CC={:?}", compiler.path()); + let mut compiler_path = compiler.path().to_owned(); + if cfg!(windows) && compiler_path.to_string_lossy().contains(' ') { + // Autoconf does not support spaces in the compiler path. This is not + // usually a problem, except on windows where the MSVC compiler (cl.exe) + // is usually found in C:\Program Files, which has a space. Fortunately, + // windows has a trick that can be used: the path can be turned into a + // "short path", turning C:\Program Files into C:\PROGRA~2. Through this + // trick, we can attempt to eliminate the spaces. + if let Some(short_path) = to_short_path(&compiler_path) { + compiler_path = short_path; + } + } + info!("CC={:?}", compiler_path); info!("CFLAGS={:?}", cflags); assert!(out_dir.exists(), "OUT_DIR does not exist"); @@ -189,7 +314,7 @@ fn main() { .replace('\\', "/"), ) .current_dir(&build_dir) - .env("CC", compiler.path()) + .env("CC", compiler_path) .env("CFLAGS", cflags.clone()) .env("LDFLAGS", cflags.clone()) .env("CPPFLAGS", cflags) @@ -308,37 +433,25 @@ fn main() { run_and_log(&mut cmd, &build_dir.join("config.log")); // Make: - let make = make_cmd(&host); - let mut make_cmd = Command::new(make); - make_cmd - .current_dir(&build_dir) + run(make_cmd(&host, &build_dir, &compiler) .arg("-j") - .arg(num_jobs.clone()); - - for (env_key, env_value) in compiler.env() { - // MSVC compilers require a handful of environment variables to be set - // to work properly, so they can find their built-in include folders and - // default library path. - make_cmd.env(env_key, env_value); - } - run(&mut make_cmd); + .arg(num_jobs.clone())); // Skip watching this environment variables to avoid rebuild in CI. if env::var("JEMALLOC_SYS_RUN_JEMALLOC_TESTS").is_ok() { info!("Building and running jemalloc tests..."); // Make tests: - run(Command::new(make) - .current_dir(&build_dir) + run(make_cmd(&host, &build_dir, &compiler) .arg("-j") .arg(num_jobs.clone()) .arg("tests")); // Run tests: - run(Command::new(make).current_dir(&build_dir).arg("check")); + run(make_cmd(&host, &build_dir, &compiler).arg("check")); } // Make install: - run(Command::new(make) + run(make_cmd(&host, &build_dir, &compiler) .current_dir(&build_dir) .arg("install_lib_static") .arg("install_include") @@ -429,7 +542,7 @@ fn gnu_target(target: &str) -> String { } } -fn make_cmd(host: &str) -> &'static str { +fn make_cmd_program(host: &str) -> &'static str { const GMAKE_HOSTS: &[&str] = &[ "bitrig", "dragonfly", @@ -447,6 +560,28 @@ fn make_cmd(host: &str) -> &'static str { } } +fn make_cmd(host: &str, build_dir: &Path, compiler: &Tool) -> Command { + let mut make = Command::new(make_cmd_program(host)); + make.current_dir(build_dir); + + for (env_key, env_value) in compiler.env() { + // MSVC compilers require a handful of environment variables to be set + // to work properly, so they can find their built-in include folders and + // default library path. + make.env(env_key, env_value); + } + + static SHELL_OVERRIDE: OnceLock> = OnceLock::new(); + + // Override SHELL if necessary. We cache the value of the SHELL to avoid + // recomputing it every time we run make. + if let Some(arg) = SHELL_OVERRIDE.get_or_init(|| find_shell_arg(host, build_dir)) { + make.arg(arg); + } + + make +} + struct BackgroundThreadSupport { always_enabled: bool, } From 9d50996c268b613a1fe8b2372bb172ca95bd828e Mon Sep 17 00:00:00 2001 From: roblabla Date: Sun, 4 Aug 2024 17:27:48 +0200 Subject: [PATCH 08/12] Disable tests on windows target Signed-off-by: roblabla --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 81d4191bb..03954a688 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: - name: x86_64-pc-windows-msvc target: x86_64-pc-windows-msvc nobgt: 0 - no_tests: 0 + no_tests: 1 tag: windows-latest - name: x86_64-apple-darwin target: x86_64-apple-darwin From 3f2e4a1886b47696f562f8a4e146c6934c296e90 Mon Sep 17 00:00:00 2001 From: roblabla Date: Sun, 4 Aug 2024 17:35:54 +0200 Subject: [PATCH 09/12] Fix clippy errors Signed-off-by: roblabla --- jemalloc-sys/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jemalloc-sys/src/lib.rs b/jemalloc-sys/src/lib.rs index 79096a0f8..078096dba 100644 --- a/jemalloc-sys/src/lib.rs +++ b/jemalloc-sys/src/lib.rs @@ -387,9 +387,9 @@ extern "C" { /// The behavior is _undefined_ if: /// /// * `size` is not in range `[req_size, alloc_size]`, where `req_size` is - /// the size requested when performing the allocation, and `alloc_size` is - /// the allocation size returned by [`nallocx`], [`sallocx`], or - /// [`xallocx`], + /// the size requested when performing the allocation, and `alloc_size` is + /// the allocation size returned by [`nallocx`], [`sallocx`], or + /// [`xallocx`], /// * `ptr` does not match a pointer earlier returned by the memory /// allocation functions of this crate, or /// * `ptr` is null, or @@ -453,8 +453,8 @@ extern "C" { /// Returns `0` on success, otherwise returns: /// /// * `EINVAL`: if `newp` is not null, and `newlen` is too large or too - /// small. Alternatively, `*oldlenp` is too large or too small; in this case - /// as much data as possible are read despite the error. + /// small. Alternatively, `*oldlenp` is too large or too small; in this case + /// as much data as possible are read despite the error. /// /// * `ENOENT`: `name` or mib specifies an unknown/invalid value. /// @@ -463,7 +463,7 @@ extern "C" { /// * `EAGAIN`: A memory allocation failure occurred. /// /// * `EFAULT`: An interface with side effects failed in some way not - /// directly related to `mallctl` read/write processing. + /// directly related to `mallctl` read/write processing. /// /// [jemalloc_mallctl]: http://jemalloc.net/jemalloc.3.html#mallctl_namespace #[cfg_attr(prefixed, link_name = "_rjem_mallctl")] From b87e39f1777ede3fb58ef3654a8511e20c6c0083 Mon Sep 17 00:00:00 2001 From: roblabla Date: Sun, 4 Aug 2024 17:46:25 +0200 Subject: [PATCH 10/12] Disable background_threads support on windows Background Threads are unsupported on windows in jemalloc, due to requiring pthreads (which aren't available on windows, at least on the msvc target). Trying to enable it will cause a runtime error causing the program to immediately abort. As such, to avoid problems, we set windows as a target that does not support background_threads. Signed-off-by: roblabla --- .github/workflows/main.yml | 2 +- jemalloc-ctl/src/lib.rs | 4 ++-- jemalloc-ctl/src/macros.rs | 6 +++--- jemalloc-sys/src/env.rs | 2 +- jemallocator/tests/background_thread_enabled.rs | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 03954a688..3a37ea047 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ jobs: include: - name: x86_64-pc-windows-msvc target: x86_64-pc-windows-msvc - nobgt: 0 + nobgt: 1 no_tests: 1 tag: windows-latest - name: x86_64-apple-darwin diff --git a/jemalloc-ctl/src/lib.rs b/jemalloc-ctl/src/lib.rs index 487f719fe..40277007c 100644 --- a/jemalloc-ctl/src/lib.rs +++ b/jemalloc-ctl/src/lib.rs @@ -141,7 +141,7 @@ option! { /// # static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; /// # /// # fn main() { - /// # #[cfg(not(target_os = "macos"))] { + /// # #[cfg(not(any(target_os = "macos", windows)))] { /// # /// use tikv_jemalloc_ctl::background_thread; /// let bg = background_thread::mib().unwrap(); @@ -171,7 +171,7 @@ option! { /// # static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; /// # /// # fn main() { - /// # #[cfg(not(target_os = "macos"))] { + /// # #[cfg(not(any(target_os = "macos", windows)))] { /// # /// use tikv_jemalloc_ctl::max_background_threads; /// let m = max_background_threads::mib().unwrap(); diff --git a/jemalloc-ctl/src/macros.rs b/jemalloc-ctl/src/macros.rs index ee4bd5ff0..02ca084b7 100644 --- a/jemalloc-ctl/src/macros.rs +++ b/jemalloc-ctl/src/macros.rs @@ -69,7 +69,7 @@ macro_rules! r { match stringify!($id) { "background_thread" | "max_background_threads" - if cfg!(target_os = "macos") => return, + if cfg!(any(target_os = "macos", windows)) => return, _ => (), } @@ -117,7 +117,7 @@ macro_rules! w { match stringify!($id) { "background_thread" | "max_background_threads" - if cfg!(target_os = "macos") => return, + if cfg!(any(target_os = "macos", windows)) => return, _ => (), } @@ -167,7 +167,7 @@ macro_rules! u { match stringify!($id) { "background_thread" | "max_background_threads" - if cfg!(target_os = "macos") => return, + if cfg!(any(target_os = "macos", windows)) => return, _ => (), } diff --git a/jemalloc-sys/src/env.rs b/jemalloc-sys/src/env.rs index 79cb55318..dd12c2197 100644 --- a/jemalloc-sys/src/env.rs +++ b/jemalloc-sys/src/env.rs @@ -12,7 +12,7 @@ pub static UNSUPPORTED_TARGETS: &[&str] = &[ pub static UNTESTED_TARGETS: &[&str] = &["openbsd", "msvc"]; /// `jemalloc`'s background_thread support is known not to work on these targets: -pub static NO_BG_THREAD_TARGETS: &[&str] = &["musl"]; +pub static NO_BG_THREAD_TARGETS: &[&str] = &["musl", "windows"]; /// targets that don't support unprefixed `malloc` // “it was found that the `realpath` function in libc would allocate with libc malloc diff --git a/jemallocator/tests/background_thread_enabled.rs b/jemallocator/tests/background_thread_enabled.rs index 76d286d6d..e37c5b222 100644 --- a/jemallocator/tests/background_thread_enabled.rs +++ b/jemallocator/tests/background_thread_enabled.rs @@ -2,7 +2,7 @@ //! library was compiled with background thread run-time support. #![cfg(feature = "background_threads_runtime_support")] #![cfg(not(feature = "unprefixed_malloc_on_supported_platforms"))] -#![cfg(not(target_env = "musl"))] +#![cfg(not(any(windows, target_env = "musl")))] use tikv_jemallocator::Jemalloc; From 52b27b46e3d4b23b9a9f22ae594feab86fb4c295 Mon Sep 17 00:00:00 2001 From: roblabla Date: Mon, 5 Aug 2024 16:06:55 +0200 Subject: [PATCH 11/12] Disable profiling tests on windows Profiling is not currently supported by jemalloc on windows, as it requires a profiling backend based on libraries available on Windows. As such, we disable the profiling-related tests when running on windows. Signed-off-by: roblabla --- jemalloc-ctl/src/macros.rs | 6 ++++++ jemalloc-ctl/src/profiling.rs | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/jemalloc-ctl/src/macros.rs b/jemalloc-ctl/src/macros.rs index 02ca084b7..ee9955f0c 100644 --- a/jemalloc-ctl/src/macros.rs +++ b/jemalloc-ctl/src/macros.rs @@ -70,6 +70,12 @@ macro_rules! r { "background_thread" | "max_background_threads" if cfg!(any(target_os = "macos", windows)) => return, + "lg_prof_interval" | + "lg_prof_sample" | + "prof_final" | + "prof_leak" | + "prof" + if cfg!(windows) => return, _ => (), } diff --git a/jemalloc-ctl/src/profiling.rs b/jemalloc-ctl/src/profiling.rs index 306ab1a8e..1d9e118e2 100644 --- a/jemalloc-ctl/src/profiling.rs +++ b/jemalloc-ctl/src/profiling.rs @@ -24,9 +24,13 @@ option! { /// # static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; /// # /// # fn main() { + /// # #[cfg(not(windows))] { + /// # /// use tikv_jemalloc_ctl::profiling; /// let lg_prof_interval = profiling::lg_prof_interval::read().unwrap(); /// println!("average interval between memory profile dumps: {}", lg_prof_interval); + /// # + /// # } // #[cfg(..)] /// # } /// ``` mib_docs: /// See [`lg_prof_interval`]. @@ -49,9 +53,13 @@ option! { /// # static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; /// # /// # fn main() { + /// # #[cfg(not(windows))] { + /// # /// use tikv_jemalloc_ctl::profiling; /// let lg_prof_sample = profiling::lg_prof_sample::read().unwrap(); /// println!("average interval between allocation samples: {}", lg_prof_sample); + /// # + /// # } // #[cfg(..)] /// # } /// ``` mib_docs: /// See [`lg_prof_sample`]. @@ -79,9 +87,13 @@ option! { /// # static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; /// # /// # fn main() { + /// # #[cfg(not(windows))] { + /// # /// use tikv_jemalloc_ctl::profiling; /// let prof_final = profiling::prof_final::read().unwrap(); /// println!("dump final memory usage to file: {}", prof_final); + /// # + /// # } // #[cfg(..)] /// # } /// ``` mib_docs: /// See [`prof_final`]. @@ -116,9 +128,13 @@ option! { /// # static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; /// # /// # fn main() { + /// # #[cfg(not(windows))] { + /// # /// use tikv_jemalloc_ctl::profiling; /// let prof = profiling::prof::read().unwrap(); /// println!("is memory profiling enabled: {}", prof); + /// # + /// # } // #[cfg(..)] /// # } /// ``` mib_docs: /// See [`prof`]. @@ -146,9 +162,13 @@ option! { /// # static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; /// # /// # fn main() { + /// # #[cfg(not(windows))] { + /// # /// use tikv_jemalloc_ctl::profiling; /// let prof_leak = profiling::prof_leak::read().unwrap(); /// println!("is leak reporting enabled: {}", prof_leak); + /// # + /// # } // #[cfg(..)] /// # } /// ``` mib_docs: /// See [`prof_leak`]. From 738cc6798009e5aeb1a75988c479dc9428aa1a9a Mon Sep 17 00:00:00 2001 From: roblabla Date: Mon, 23 Sep 2024 15:43:46 +0200 Subject: [PATCH 12/12] Fix compiling jemalloc with MSVC 2022 Signed-off-by: roblabla --- jemalloc-sys/configure/configure | 25 ++++++++++++++++++++----- jemalloc-sys/jemalloc | 2 +- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/jemalloc-sys/configure/configure b/jemalloc-sys/configure/configure index 7c4e6e266..0d8d4b7c9 100755 --- a/jemalloc-sys/configure/configure +++ b/jemalloc-sys/configure/configure @@ -8615,7 +8615,21 @@ _ACEOF -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5 + +# On MSVC, log is an intrinsic that doesn't require libm. However, +# AC_SEARCH_LIBS does not successfully detect this, as it will try to compile +# a program using the wrong signature for log. Newer versions of MSVC CL detects +# this and rejects the program with the following messages. +# +# conftest.c(40): warning C4391: 'char log()': incorrect return type for intrinsic function, expected 'double' +# conftest.c(44): error C2168: 'log': too few actual parameters for intrinsic function +# +# Since log is always available on MSVC (it's been around since the dawn of +# time), we simply always assume it's there if MSVC is detected. +if test "x$je_cv_msvc" = "xyes" ; then + LM= +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5 $as_echo_n "checking for library containing log... " >&6; } if ${ac_cv_search_log+:} false; then : $as_echo_n "(cached) " >&6 @@ -8673,10 +8687,11 @@ else as_fn_error $? "Missing math functions" "$LINENO" 5 fi -if test "x$ac_cv_search_log" != "xnone required" ; then - LM="$ac_cv_search_log" -else - LM= + if test "x$ac_cv_search_log" != "xnone required" ; then + LM="$ac_cv_search_log" + else + LM= + fi fi diff --git a/jemalloc-sys/jemalloc b/jemalloc-sys/jemalloc index e13ca993e..2f15f126a 160000 --- a/jemalloc-sys/jemalloc +++ b/jemalloc-sys/jemalloc @@ -1 +1 @@ -Subproject commit e13ca993e8ccb9ba9847cc330696e02839f328f7 +Subproject commit 2f15f126a028d78bd089e331e603e60229b7558a