diff --git a/.ci/scripts/unittest-macos-cmake.sh b/.ci/scripts/unittest-macos-cmake.sh
index cdb40c40244..1a6cd2a15f2 100755
--- a/.ci/scripts/unittest-macos-cmake.sh
+++ b/.ci/scripts/unittest-macos-cmake.sh
@@ -11,3 +11,4 @@ ${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml
 # Run gtest
 LLVM_PROFDATA="xcrun llvm-profdata" LLVM_COV="xcrun llvm-cov" \
 ${CONDA_RUN} test/run_oss_cpp_tests.sh
+${CONDA_RUN} test/check_for_installed_private_headers_in_cmake_out.sh
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9aa53004b03..357e9039b0f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -485,24 +485,29 @@ install(
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/runtime/core
   FILES_MATCHING
   PATTERN "*.h"
+  PATTERN "testing_util" EXCLUDE
 )
 install(
   DIRECTORY runtime/executor/
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/runtime/executor
   FILES_MATCHING
   PATTERN "*.h"
+  PATTERN "test" EXCLUDE
+  PATTERN "platform_memory_allocator.h" EXCLUDE
 )
 install(
   DIRECTORY runtime/kernel/
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/runtime/kernel
   FILES_MATCHING
   PATTERN "*.h"
+  PATTERN "test" EXCLUDE
 )
 install(
   DIRECTORY runtime/platform/
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/runtime/platform
   FILES_MATCHING
   PATTERN "*.h"
+  PATTERN "test" EXCLUDE
 )
 install(
   DIRECTORY extension/kernel_util/
@@ -587,11 +592,15 @@ endif()
 
 if(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/data_loader)
+  if(NOT WIN32)
+    set(data_loader_exclude_pattern "*mman_windows.h")
+  endif()
   install(
     DIRECTORY extension/data_loader/
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/extension/data_loader
     FILES_MATCHING
     PATTERN "*.h"
+    PATTERN ${data_loader_exclude_pattern} EXCLUDE
   )
   list(APPEND _executorch_extensions extension_data_loader)
 endif()
diff --git a/test/check_for_installed_private_headers_in_cmake_out.sh b/test/check_for_installed_private_headers_in_cmake_out.sh
new file mode 100755
index 00000000000..a7e5034196e
--- /dev/null
+++ b/test/check_for_installed_private_headers_in_cmake_out.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# This script verifies that all headers that are installed under
+# cmake-out/include/executorch are exported_headers of some Buck
+# target. (It does *not* verify the reverse, namely that all
+# exported_headers of every Buck target that should have been built
+# when that directory was installed are actually installed.)
+#
+# Ideally, "some Buck target" would include any target in the whole
+# repo, but we cannot yet buck query the whole repo. (See
+# .ci/scripts/unittest-buck2.sh.) Instead, we query a manually-curated
+# list of targets.
+
+set -euxo pipefail
+
+BUCK_HEADERS_TEMPFILE=$(mktemp /tmp/check_private_headers_buck.txt.XXXXXX)
+ACTUAL_HEADERS_TEMPFILE=$(mktemp /tmp/check_private_headers_installed.txt.XXXXXX)
+SOURCE_ROOT_DIR=$(git rev-parse --show-toplevel)
+BUCK2=$(python3 "${SOURCE_ROOT_DIR}/tools/cmake/resolve_buck.py" --cache_dir="${SOURCE_ROOT_DIR}/buck2-bin")
+if [[ "$BUCK2" == "buck2" ]]; then
+  BUCK2=$(command -v buck2)
+fi
+
+"${SOURCE_ROOT_DIR}/scripts/print_exported_headers.py" \
+    --buck2=$(realpath "$BUCK2") --targets \
+    //extension/data_loader: //extension/evalue_util: \
+    //extension/flat_tensor: //extension/llm/runner: //extension/kernel_util: //extension/module: \
+    //extension/runner_util: //extension/tensor: //extension/threadpool: \
+    | sort > "${BUCK_HEADERS_TEMPFILE}"
+find "${SOURCE_ROOT_DIR}/cmake-out/include/executorch" -name '*.h' | \
+    sed -e "s|${SOURCE_ROOT_DIR}/cmake-out/include/executorch/||" | \
+    # Don't complain about generated Functions.h \
+    grep -E -v 'Functions.h$' |  sort > "${ACTUAL_HEADERS_TEMPFILE}"
+ACTUAL_HEADERS_NOT_EXPORTED_IN_BUCK=$(comm -13 "${BUCK_HEADERS_TEMPFILE}" "${ACTUAL_HEADERS_TEMPFILE}")
+if [[ -n "${ACTUAL_HEADERS_NOT_EXPORTED_IN_BUCK}" ]]; then
+    >&2 echo "The following non-exported headers were installed:
+${ACTUAL_HEADERS_NOT_EXPORTED_IN_BUCK}"
+    exit 1
+fi