From 07adf1a07ae07c2c4ea6c5da78b4ec09bc50c8bf Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Fri, 7 Feb 2025 17:36:57 -0800 Subject: [PATCH 01/29] Update [ghstack-poisoned] --- build/Codegen.cmake | 21 ++ build/build_variables.bzl | 495 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 516 insertions(+) create mode 100644 build/build_variables.bzl diff --git a/build/Codegen.cmake b/build/Codegen.cmake index 435b3d24802..0d93590be99 100644 --- a/build/Codegen.cmake +++ b/build/Codegen.cmake @@ -213,3 +213,24 @@ function(merge_yaml) WORKING_DIRECTORY ${EXECUTORCH_ROOT} ) endfunction() + +function(append_filelist name outputvar) + set(_rootdir "${EXECUTORCH_ROOT}/") + # configure_file adds its input to the list of CMAKE_RERUN dependencies + configure_file( + ${PROJECT_SOURCE_DIR}/build/build_variables.bzl + ${PROJECT_BINARY_DIR}/build_variables.bzl + COPYONLY) + execute_process( + COMMAND "${Python_EXECUTABLE}" -c + "exec(open('${PROJECT_SOURCE_DIR}/build/build_variables.bzl').read());print(';'.join(['${_rootdir}' + x for x in ${name}]))" + WORKING_DIRECTORY "${_rootdir}" + RESULT_VARIABLE _retval + OUTPUT_VARIABLE _tempvar) + if(NOT _retval EQUAL 0) + message(FATAL_ERROR "Failed to fetch filelist ${name} from build_variables.bzl") + endif() + string(REPLACE "\n" "" _tempvar "${_tempvar}") + list(APPEND ${outputvar} ${_tempvar}) + set(${outputvar} "${${outputvar}}" PARENT_SCOPE) +endfunction() diff --git a/build/build_variables.bzl b/build/build_variables.bzl new file mode 100644 index 00000000000..75f1792f6fa --- /dev/null +++ b/build/build_variables.bzl @@ -0,0 +1,495 @@ +# WARNING: the contents of this file must BOTH be valid Starlark (for Buck) as well as +# valid Python (for our cmake build). This means that load() directives are not allowed +# (as they are not recognized by Python). If you want to fix this, figure out how run +# this file from cmake with a proper Starlark interpreter as part of the default OSS +# build process. If you need some nontrivial Starlark features, make a separate bzl +# file. (Remember that bzl files are not exported via ShipIt by default, so you may also +# need to update ExecuTorch's ShipIt config.) + +# _executorch__srcs +EXECUTORCH_SRCS = [ + "kernels/prim_ops/et_copy_index.cpp", + "kernels/prim_ops/et_view.cpp", + "kernels/prim_ops/register_prim_ops.cpp", +] + +# _executorch_core__srcs +EXECUTORCH_CORE_SRCS = [ + "runtime/backend/interface.cpp", + "runtime/core/evalue.cpp", + "runtime/core/exec_aten/util/tensor_shape_to_c_string.cpp", + "runtime/core/exec_aten/util/tensor_util_portable.cpp", + "runtime/core/portable_type/tensor_impl.cpp", + "runtime/core/tag.cpp", + "runtime/executor/method.cpp", + "runtime/executor/method_meta.cpp", + "runtime/executor/program.cpp", + "runtime/executor/tensor_parser_exec_aten.cpp", + "runtime/executor/tensor_parser_portable.cpp", + "runtime/kernel/operator_registry.cpp", + "runtime/platform/abort.cpp", + "runtime/platform/default/posix.cpp", + "runtime/platform/log.cpp", + "runtime/platform/profiler.cpp", + "runtime/platform/runtime.cpp", + "schema/extended_header.cpp", +] + +# _portable_kernels__srcs +PORTABLE_KERNELS_SRCS = [ + "kernels/portable/cpu/op__empty_dim_order.cpp", + "kernels/portable/cpu/op__to_dim_order_copy.cpp", + "kernels/portable/cpu/op_abs.cpp", + "kernels/portable/cpu/op_acos.cpp", + "kernels/portable/cpu/op_acosh.cpp", + "kernels/portable/cpu/op_add.cpp", + "kernels/portable/cpu/op_addmm.cpp", + "kernels/portable/cpu/op_alias_copy.cpp", + "kernels/portable/cpu/op_allclose.cpp", + "kernels/portable/cpu/op_amax.cpp", + "kernels/portable/cpu/op_amin.cpp", + "kernels/portable/cpu/op_any.cpp", + "kernels/portable/cpu/op_arange.cpp", + "kernels/portable/cpu/op_argmax.cpp", + "kernels/portable/cpu/op_argmin.cpp", + "kernels/portable/cpu/op_as_strided_copy.cpp", + "kernels/portable/cpu/op_asin.cpp", + "kernels/portable/cpu/op_asinh.cpp", + "kernels/portable/cpu/op_atan.cpp", + "kernels/portable/cpu/op_atan2.cpp", + "kernels/portable/cpu/op_atanh.cpp", + "kernels/portable/cpu/op_avg_pool2d.cpp", + "kernels/portable/cpu/op_bitwise_and.cpp", + "kernels/portable/cpu/op_bitwise_not.cpp", + "kernels/portable/cpu/op_bitwise_or.cpp", + "kernels/portable/cpu/op_bitwise_xor.cpp", + "kernels/portable/cpu/op_bmm.cpp", + "kernels/portable/cpu/op_cat.cpp", + "kernels/portable/cpu/op_cdist_forward.cpp", + "kernels/portable/cpu/op_ceil.cpp", + "kernels/portable/cpu/op_clamp.cpp", + "kernels/portable/cpu/op_clone.cpp", + "kernels/portable/cpu/op_constant_pad_nd.cpp", + "kernels/portable/cpu/op_convolution.cpp", + "kernels/portable/cpu/op_convolution_backward.cpp", + "kernels/portable/cpu/op_copy.cpp", + "kernels/portable/cpu/op_cos.cpp", + "kernels/portable/cpu/op_cosh.cpp", + "kernels/portable/cpu/op_cumsum.cpp", + "kernels/portable/cpu/op_detach_copy.cpp", + "kernels/portable/cpu/op_diagonal_copy.cpp", + "kernels/portable/cpu/op_div.cpp", + "kernels/portable/cpu/op_embedding.cpp", + "kernels/portable/cpu/op_empty.cpp", + "kernels/portable/cpu/op_eq.cpp", + "kernels/portable/cpu/op_erf.cpp", + "kernels/portable/cpu/op_exp.cpp", + "kernels/portable/cpu/op_expand_copy.cpp", + "kernels/portable/cpu/op_expm1.cpp", + "kernels/portable/cpu/op_fill.cpp", + "kernels/portable/cpu/op_flip.cpp", + "kernels/portable/cpu/op_floor.cpp", + "kernels/portable/cpu/op_floor_divide.cpp", + "kernels/portable/cpu/op_fmod.cpp", + "kernels/portable/cpu/op_full.cpp", + "kernels/portable/cpu/op_full_like.cpp", + "kernels/portable/cpu/op_gather.cpp", + "kernels/portable/cpu/op_ge.cpp", + "kernels/portable/cpu/op_gelu.cpp", + "kernels/portable/cpu/op_glu.cpp", + "kernels/portable/cpu/op_gt.cpp", + "kernels/portable/cpu/op_hardtanh.cpp", + "kernels/portable/cpu/op_index.cpp", + "kernels/portable/cpu/op_index_put.cpp", + "kernels/portable/cpu/op_index_select.cpp", + "kernels/portable/cpu/op_isinf.cpp", + "kernels/portable/cpu/op_isnan.cpp", + "kernels/portable/cpu/op_le.cpp", + "kernels/portable/cpu/op_leaky_relu.cpp", + "kernels/portable/cpu/op_lift_fresh_copy.cpp", + "kernels/portable/cpu/op_linear_scratch_example.cpp", + "kernels/portable/cpu/op_log.cpp", + "kernels/portable/cpu/op_log10.cpp", + "kernels/portable/cpu/op_log1p.cpp", + "kernels/portable/cpu/op_log2.cpp", + "kernels/portable/cpu/op_log_softmax.cpp", + "kernels/portable/cpu/op_logical_and.cpp", + "kernels/portable/cpu/op_logical_not.cpp", + "kernels/portable/cpu/op_logical_or.cpp", + "kernels/portable/cpu/op_logical_xor.cpp", + "kernels/portable/cpu/op_logit.cpp", + "kernels/portable/cpu/op_lt.cpp", + "kernels/portable/cpu/op_masked_fill.cpp", + "kernels/portable/cpu/op_masked_scatter.cpp", + "kernels/portable/cpu/op_masked_select.cpp", + "kernels/portable/cpu/op_max.cpp", + "kernels/portable/cpu/op_max_pool2d_with_indices.cpp", + "kernels/portable/cpu/op_maximum.cpp", + "kernels/portable/cpu/op_mean.cpp", + "kernels/portable/cpu/op_min.cpp", + "kernels/portable/cpu/op_minimum.cpp", + "kernels/portable/cpu/op_mm.cpp", + "kernels/portable/cpu/op_mul.cpp", + "kernels/portable/cpu/op_narrow_copy.cpp", + "kernels/portable/cpu/op_native_batch_norm.cpp", + "kernels/portable/cpu/op_native_group_norm.cpp", + "kernels/portable/cpu/op_native_layer_norm.cpp", + "kernels/portable/cpu/op_ne.cpp", + "kernels/portable/cpu/op_neg.cpp", + "kernels/portable/cpu/op_nonzero.cpp", + "kernels/portable/cpu/op_ones.cpp", + "kernels/portable/cpu/op_pdist_forward.cpp", + "kernels/portable/cpu/op_permute_copy.cpp", + "kernels/portable/cpu/op_pixel_shuffle.cpp", + "kernels/portable/cpu/op_pixel_unshuffle.cpp", + "kernels/portable/cpu/op_pow.cpp", + "kernels/portable/cpu/op_prod.cpp", + "kernels/portable/cpu/op_reciprocal.cpp", + "kernels/portable/cpu/op_reflection_pad1d.cpp", + "kernels/portable/cpu/op_reflection_pad2d.cpp", + "kernels/portable/cpu/op_reflection_pad3d.cpp", + "kernels/portable/cpu/op_relu.cpp", + "kernels/portable/cpu/op_remainder.cpp", + "kernels/portable/cpu/op_repeat.cpp", + "kernels/portable/cpu/op_repeat_interleave.cpp", + "kernels/portable/cpu/op_replication_pad1d.cpp", + "kernels/portable/cpu/op_replication_pad2d.cpp", + "kernels/portable/cpu/op_replication_pad3d.cpp", + "kernels/portable/cpu/op_roll.cpp", + "kernels/portable/cpu/op_round.cpp", + "kernels/portable/cpu/op_rsqrt.cpp", + "kernels/portable/cpu/op_rsub.cpp", + "kernels/portable/cpu/op_scalar_tensor.cpp", + "kernels/portable/cpu/op_scatter.cpp", + "kernels/portable/cpu/op_scatter_add.cpp", + "kernels/portable/cpu/op_select_copy.cpp", + "kernels/portable/cpu/op_select_scatter.cpp", + "kernels/portable/cpu/op_sigmoid.cpp", + "kernels/portable/cpu/op_sign.cpp", + "kernels/portable/cpu/op_sin.cpp", + "kernels/portable/cpu/op_sinh.cpp", + "kernels/portable/cpu/op_slice_copy.cpp", + "kernels/portable/cpu/op_slice_scatter.cpp", + "kernels/portable/cpu/op_softmax.cpp", + "kernels/portable/cpu/op_split_copy.cpp", + "kernels/portable/cpu/op_split_with_sizes_copy.cpp", + "kernels/portable/cpu/op_sqrt.cpp", + "kernels/portable/cpu/op_squeeze_copy.cpp", + "kernels/portable/cpu/op_stack.cpp", + "kernels/portable/cpu/op_sub.cpp", + "kernels/portable/cpu/op_sum.cpp", + "kernels/portable/cpu/op_t_copy.cpp", + "kernels/portable/cpu/op_tan.cpp", + "kernels/portable/cpu/op_tanh.cpp", + "kernels/portable/cpu/op_to_copy.cpp", + "kernels/portable/cpu/op_topk.cpp", + "kernels/portable/cpu/op_transpose_copy.cpp", + "kernels/portable/cpu/op_tril.cpp", + "kernels/portable/cpu/op_trunc.cpp", + "kernels/portable/cpu/op_unbind_copy.cpp", + "kernels/portable/cpu/op_unsqueeze_copy.cpp", + "kernels/portable/cpu/op_upsample_bilinear2d.cpp", + "kernels/portable/cpu/op_upsample_nearest2d.cpp", + "kernels/portable/cpu/op_var.cpp", + "kernels/portable/cpu/op_view_copy.cpp", + "kernels/portable/cpu/op_where.cpp", + "kernels/portable/cpu/op_zeros.cpp", + "kernels/portable/cpu/pattern/unary_ufunc_realh.cpp", + "kernels/portable/cpu/pattern/unary_ufunc_realhb_to_bool.cpp", + "kernels/portable/cpu/pattern/unary_ufunc_realhbbf16_to_floathbf16.cpp", + "kernels/portable/cpu/util/activation_ops_util.cpp", + "kernels/portable/cpu/util/advanced_index_util.cpp", + "kernels/portable/cpu/util/broadcast_util.cpp", + "kernels/portable/cpu/util/copy_ops_util.cpp", + "kernels/portable/cpu/util/distance_util.cpp", + "kernels/portable/cpu/util/dtype_util.cpp", + "kernels/portable/cpu/util/index_util.cpp", + "kernels/portable/cpu/util/kernel_ops_util.cpp", + "kernels/portable/cpu/util/matmul_ops_util.cpp", + "kernels/portable/cpu/util/normalization_ops_util.cpp", + "kernels/portable/cpu/util/padding_util.cpp", + "kernels/portable/cpu/util/reduce_util.cpp", + "kernels/portable/cpu/util/repeat_util.cpp", + "kernels/portable/cpu/util/select_copy_util.cpp", + "kernels/portable/cpu/util/slice_util.cpp", + "kernels/portable/cpu/util/upsample_util.cpp", +] + +# _optimized_kernels__srcs +OPTIMIZED_KERNELS_SRCS = [ + "extension/parallel/thread_parallel.cpp", + "kernels/optimized/blas/BlasKernel.cpp", + "kernels/optimized/blas/CPUBlas.cpp", + "kernels/optimized/cpu/op_add.cpp", + "kernels/optimized/cpu/op_bmm.cpp", + "kernels/optimized/cpu/op_div.cpp", + "kernels/optimized/cpu/op_exp.cpp", + "kernels/optimized/cpu/op_fft_r2c.cpp", + "kernels/optimized/cpu/op_le.cpp", + "kernels/optimized/cpu/op_linear.cpp", + "kernels/optimized/cpu/op_mm.cpp", + "kernels/optimized/cpu/op_mul.cpp", + "kernels/optimized/cpu/op_native_layer_norm.cpp", + "kernels/optimized/cpu/op_neg.cpp", + "kernels/optimized/cpu/op_sigmoid.cpp", + "kernels/optimized/cpu/op_sub.cpp", +] + +# _quantized_kernels__srcs +QUANTIZED_KERNELS_SRCS = [ + "kernels/quantized/cpu/embeddingxb.cpp", + "kernels/quantized/cpu/op_add.cpp", + "kernels/quantized/cpu/op_choose_qparams.cpp", + "kernels/quantized/cpu/op_dequantize.cpp", + "kernels/quantized/cpu/op_embedding.cpp", + "kernels/quantized/cpu/op_embedding2b.cpp", + "kernels/quantized/cpu/op_embedding4b.cpp", + "kernels/quantized/cpu/op_mixed_linear.cpp", + "kernels/quantized/cpu/op_mixed_mm.cpp", + "kernels/quantized/cpu/op_quantize.cpp", +] + +# _program_schema__srcs +PROGRAM_SCHEMA_SRCS = [ + "schema/program.fbs", + "schema/scalar_type.fbs", +] + +# _optimized_cpublas__srcs +OPTIMIZED_CPUBLAS_SRCS = [ + "extension/parallel/thread_parallel.cpp", + "extension/threadpool/threadpool.cpp", + "extension/threadpool/threadpool_guard.cpp", + "kernels/optimized/blas/BlasKernel.cpp", + "kernels/optimized/blas/CPUBlas.cpp", +] + +# _optimized_native_cpu_ops_oss__srcs +OPTIMIZED_NATIVE_CPU_OPS_OSS_SRCS = [ + "codegen/templates/RegisterCodegenUnboxedKernels.cpp", + "codegen/templates/RegisterDispatchKeyCustomOps.cpp", + "codegen/templates/RegisterKernels.cpp", + "codegen/templates/RegisterSchema.cpp", + "extension/parallel/thread_parallel.cpp", + "extension/threadpool/threadpool.cpp", + "extension/threadpool/threadpool_guard.cpp", + "kernels/optimized/blas/BlasKernel.cpp", + "kernels/optimized/blas/CPUBlas.cpp", + "kernels/optimized/cpu/op_add.cpp", + "kernels/optimized/cpu/op_bmm.cpp", + "kernels/optimized/cpu/op_div.cpp", + "kernels/optimized/cpu/op_exp.cpp", + "kernels/optimized/cpu/op_fft_r2c.cpp", + "kernels/optimized/cpu/op_le.cpp", + "kernels/optimized/cpu/op_linear.cpp", + "kernels/optimized/cpu/op_mm.cpp", + "kernels/optimized/cpu/op_mul.cpp", + "kernels/optimized/cpu/op_native_layer_norm.cpp", + "kernels/optimized/cpu/op_neg.cpp", + "kernels/optimized/cpu/op_sigmoid.cpp", + "kernels/optimized/cpu/op_sub.cpp", +] + +# _extension_data_loader__srcs +EXTENSION_DATA_LOADER_SRCS = [ + "extension/data_loader/file_data_loader.cpp", + "extension/data_loader/mmap_data_loader.cpp", +] + +# _extension_module__srcs +EXTENSION_MODULE_SRCS = [ + "extension/module/module.cpp", +] + +# _extension_runner_util__srcs +EXTENSION_RUNNER_UTIL_SRCS = [ + "extension/runner_util/inputs.cpp", + "extension/runner_util/inputs_portable.cpp", +] + +# _extension_llm_runner__srcs +EXTENSION_LLM_RUNNER_SRCS = [ + "extension/data_loader/file_data_loader.cpp", + "extension/data_loader/mmap_data_loader.cpp", + "extension/llm/runner/text_decoder_runner.cpp", + "extension/llm/runner/text_prefiller.cpp", + "extension/llm/sampler/sampler.cpp", + "extension/tensor/tensor_ptr.cpp", + "extension/tensor/tensor_ptr_maker.cpp", +] + +# _extension_tensor__srcs +EXTENSION_TENSOR_SRCS = [ + "extension/tensor/tensor_ptr.cpp", + "extension/tensor/tensor_ptr_maker.cpp", +] + +# _extension_threadpool__srcs +EXTENSION_THREADPOOL_SRCS = [ + "extension/threadpool/threadpool.cpp", + "extension/threadpool/threadpool_guard.cpp", +] + +# _extension_training__srcs +EXTENSION_TRAINING_SRCS = [ + "extension/data_loader/file_data_loader.cpp", + "extension/data_loader/mmap_data_loader.cpp", + "extension/module/module.cpp", + "extension/training/module/training_module.cpp", + "extension/training/optimizer/sgd.cpp", + "kernels/prim_ops/et_copy_index.cpp", + "kernels/prim_ops/et_view.cpp", + "kernels/prim_ops/register_prim_ops.cpp", +] + +# _train_xor__srcs +TRAIN_XOR_SRCS = [ + "extension/data_loader/file_data_loader.cpp", + "extension/data_loader/mmap_data_loader.cpp", + "extension/flat_tensor/serialize/flat_tensor_header.cpp", + "extension/flat_tensor/serialize/serialize.cpp", + "extension/module/module.cpp", + "extension/tensor/tensor_ptr.cpp", + "extension/tensor/tensor_ptr_maker.cpp", + "extension/training/examples/XOR/train.cpp", + "extension/training/module/training_module.cpp", + "extension/training/optimizer/sgd.cpp", +] + +# _executor_runner__srcs +EXECUTOR_RUNNER_SRCS = [ + "examples/portable/executor_runner/executor_runner.cpp", + "extension/data_loader/file_data_loader.cpp", + "extension/evalue_util/print_evalue.cpp", + "extension/runner_util/inputs.cpp", + "extension/runner_util/inputs_portable.cpp", + "runtime/executor/test/test_backend_compiler_lib.cpp", +] + +# _size_test__srcs +SIZE_TEST_SRCS = [ + "extension/data_loader/file_data_loader.cpp", + "test/size_test.cpp", +] + +# _mps_executor_runner__srcs +MPS_EXECUTOR_RUNNER_SRCS = [ + "backends/apple/mps/runtime/MPSBackend.mm", + "backends/apple/mps/runtime/MPSCompiler.mm", + "backends/apple/mps/runtime/MPSDelegateHeader.mm", + "backends/apple/mps/runtime/MPSDevice.mm", + "backends/apple/mps/runtime/MPSExecutor.mm", + "backends/apple/mps/runtime/MPSGraphBuilder.mm", + "backends/apple/mps/runtime/MPSStream.mm", + "backends/apple/mps/runtime/operations/ActivationOps.mm", + "backends/apple/mps/runtime/operations/BinaryOps.mm", + "backends/apple/mps/runtime/operations/ClampOps.mm", + "backends/apple/mps/runtime/operations/ConstantOps.mm", + "backends/apple/mps/runtime/operations/ConvolutionOps.mm", + "backends/apple/mps/runtime/operations/IndexingOps.mm", + "backends/apple/mps/runtime/operations/LinearAlgebra.mm", + "backends/apple/mps/runtime/operations/NormalizationOps.mm", + "backends/apple/mps/runtime/operations/OperationUtils.mm", + "backends/apple/mps/runtime/operations/PadOps.mm", + "backends/apple/mps/runtime/operations/PoolingOps.mm", + "backends/apple/mps/runtime/operations/QuantDequant.mm", + "backends/apple/mps/runtime/operations/RangeOps.mm", + "backends/apple/mps/runtime/operations/ReduceOps.mm", + "backends/apple/mps/runtime/operations/ShapeOps.mm", + "backends/apple/mps/runtime/operations/UnaryOps.mm", + "devtools/bundled_program/bundled_program.cpp", + "devtools/etdump/emitter.cpp", + "devtools/etdump/etdump_flatcc.cpp", + "examples/apple/mps/executor_runner/mps_executor_runner.mm", + "extension/data_loader/file_data_loader.cpp", + "extension/evalue_util/print_evalue.cpp", + "extension/runner_util/inputs.cpp", + "extension/runner_util/inputs_portable.cpp", +] + +# _mps_backend__srcs +MPS_BACKEND_SRCS = [ + "backends/apple/mps/runtime/MPSBackend.mm", + "backends/apple/mps/runtime/MPSCompiler.mm", + "backends/apple/mps/runtime/MPSDelegateHeader.mm", + "backends/apple/mps/runtime/MPSDevice.mm", + "backends/apple/mps/runtime/MPSExecutor.mm", + "backends/apple/mps/runtime/MPSGraphBuilder.mm", + "backends/apple/mps/runtime/MPSStream.mm", + "backends/apple/mps/runtime/operations/ActivationOps.mm", + "backends/apple/mps/runtime/operations/BinaryOps.mm", + "backends/apple/mps/runtime/operations/ClampOps.mm", + "backends/apple/mps/runtime/operations/ConstantOps.mm", + "backends/apple/mps/runtime/operations/ConvolutionOps.mm", + "backends/apple/mps/runtime/operations/IndexingOps.mm", + "backends/apple/mps/runtime/operations/LinearAlgebra.mm", + "backends/apple/mps/runtime/operations/NormalizationOps.mm", + "backends/apple/mps/runtime/operations/OperationUtils.mm", + "backends/apple/mps/runtime/operations/PadOps.mm", + "backends/apple/mps/runtime/operations/PoolingOps.mm", + "backends/apple/mps/runtime/operations/QuantDequant.mm", + "backends/apple/mps/runtime/operations/RangeOps.mm", + "backends/apple/mps/runtime/operations/ReduceOps.mm", + "backends/apple/mps/runtime/operations/ShapeOps.mm", + "backends/apple/mps/runtime/operations/UnaryOps.mm", +] + +# _mps_schema__srcs +MPS_SCHEMA_SRCS = [ + "backends/apple/mps/serialization/schema.fbs", +] + +# _xnn_executor_runner__srcs +XNN_EXECUTOR_RUNNER_SRCS = [ + "examples/portable/executor_runner/executor_runner.cpp", + "extension/data_loader/file_data_loader.cpp", + "extension/evalue_util/print_evalue.cpp", + "extension/runner_util/inputs.cpp", + "extension/runner_util/inputs_portable.cpp", +] + +# _xnnpack_backend__srcs +XNNPACK_BACKEND_SRCS = [ + "backends/xnnpack/runtime/XNNCompiler.cpp", + "backends/xnnpack/runtime/XNNExecutor.cpp", + "backends/xnnpack/runtime/XNNHeader.cpp", + "backends/xnnpack/runtime/XNNPACKBackend.cpp", + "backends/xnnpack/runtime/profiling/XNNProfiler.cpp", + "extension/threadpool/threadpool.cpp", + "extension/threadpool/threadpool_guard.cpp", +] + +# _xnnpack_schema__srcs +XNNPACK_SCHEMA_SRCS = [ + "backends/xnnpack/serialization/runtime_schema.fbs", +] + +# _vulkan_schema__srcs +VULKAN_SCHEMA_SRCS = [ + "backends/vulkan/serialization/schema.fbs", +] + +# _custom_ops__srcs +CUSTOM_OPS_SRCS = [ + "extension/llm/custom_ops/op_fallback.cpp", + "extension/llm/custom_ops/op_fast_hadamard_transform.cpp", + "extension/llm/custom_ops/op_sdpa.cpp", + "extension/llm/custom_ops/op_update_cache.cpp", + "extension/llm/custom_ops/spinquant/fast_hadamard_transform.cpp", + "extension/llm/custom_ops/spinquant/third-party/FFHT/fht_neon.c", + "kernels/portable/cpu/util/reduce_util.cpp", +] + +# _llama_runner__srcs +LLAMA_RUNNER_SRCS = [ + "examples/models/llama/runner/runner.cpp", + "examples/models/llama/tokenizer/llama_tiktoken.cpp", + "extension/evalue_util/print_evalue.cpp", + "extension/llm/runner/text_decoder_runner.cpp", + "extension/llm/runner/text_prefiller.cpp", + "extension/llm/sampler/sampler.cpp", + "extension/llm/tokenizer/bpe_tokenizer.cpp", + "extension/llm/tokenizer/tiktoken.cpp", + "extension/tensor/tensor_ptr.cpp", + "extension/tensor/tensor_ptr_maker.cpp", +] From bc3fd39b63e0b7dd9d9e37245255018a1720c0c6 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Sat, 8 Feb 2025 19:07:37 -0800 Subject: [PATCH 02/29] Update [ghstack-poisoned] --- CMakeLists.txt | 2 + build/Codegen.cmake | 107 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 100 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ca8d1bbbcf2..09fa8ea74f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,7 @@ cmake_minimum_required(VERSION 3.19) project(executorch) +include(build/Codegen.cmake) include(build/Utils.cmake) include(CMakeDependentOption) @@ -384,6 +385,7 @@ if(NOT EXECUTORCH_SRCS_FILE) message(STATUS "executorch: Generating source lists") set(EXECUTORCH_SRCS_FILE "${CMAKE_CURRENT_BINARY_DIR}/executorch_srcs.cmake") extract_sources(${EXECUTORCH_SRCS_FILE}) + validate_build_variables() endif() # This file defines the `___srcs` variables used below. diff --git a/build/Codegen.cmake b/build/Codegen.cmake index 0d93590be99..45f5d48e391 100644 --- a/build/Codegen.cmake +++ b/build/Codegen.cmake @@ -215,22 +215,111 @@ function(merge_yaml) endfunction() function(append_filelist name outputvar) - set(_rootdir "${EXECUTORCH_ROOT}/") # configure_file adds its input to the list of CMAKE_RERUN dependencies configure_file( - ${PROJECT_SOURCE_DIR}/build/build_variables.bzl - ${PROJECT_BINARY_DIR}/build_variables.bzl - COPYONLY) + ${PROJECT_SOURCE_DIR}/build/build_variables.bzl + ${PROJECT_BINARY_DIR}/build_variables.bzl COPYONLY + ) execute_process( - COMMAND "${Python_EXECUTABLE}" -c - "exec(open('${PROJECT_SOURCE_DIR}/build/build_variables.bzl').read());print(';'.join(['${_rootdir}' + x for x in ${name}]))" + COMMAND + "${PYTHON_EXECUTABLE}" -c + "exec(open('${PROJECT_SOURCE_DIR}/build/build_variables.bzl').read());print(';'.join(${name}))" WORKING_DIRECTORY "${_rootdir}" RESULT_VARIABLE _retval - OUTPUT_VARIABLE _tempvar) + OUTPUT_VARIABLE _tempvar + ERROR_VARIABLE _stderr + ) if(NOT _retval EQUAL 0) - message(FATAL_ERROR "Failed to fetch filelist ${name} from build_variables.bzl") + message( + FATAL_ERROR + "Failed to fetch filelist ${name} from build_variables.bzl with output ${_tempvar} and stderr ${_stderr}" + ) endif() string(REPLACE "\n" "" _tempvar "${_tempvar}") list(APPEND ${outputvar} ${_tempvar}) - set(${outputvar} "${${outputvar}}" PARENT_SCOPE) + set(${outputvar} + "${${outputvar}}" + PARENT_SCOPE + ) +endfunction() + +function(validate_build_variables) + include(${EXECUTORCH_SRCS_FILE}) + set(BUILD_VARIABLES_FILELISTS + EXECUTORCH_SRCS + EXECUTORCH_CORE_SRCS + PORTABLE_KERNELS_SRCS + OPTIMIZED_KERNELS_SRCS + QUANTIZED_KERNELS_SRCS + PROGRAM_SCHEMA_SRCS + OPTIMIZED_CPUBLAS_SRCS + OPTIMIZED_NATIVE_CPU_OPS_OSS_SRCS + EXTENSION_DATA_LOADER_SRCS + EXTENSION_MODULE_SRCS + EXTENSION_RUNNER_UTIL_SRCS + EXTENSION_LLM_RUNNER_SRCS + EXTENSION_TENSOR_SRCS + EXTENSION_THREADPOOL_SRCS + EXTENSION_TRAINING_SRCS + TRAIN_XOR_SRCS + EXECUTOR_RUNNER_SRCS + SIZE_TEST_SRCS + MPS_EXECUTOR_RUNNER_SRCS + MPS_BACKEND_SRCS + MPS_SCHEMA_SRCS + XNN_EXECUTOR_RUNNER_SRCS + XNNPACK_BACKEND_SRCS + XNNPACK_SCHEMA_SRCS + VULKAN_SCHEMA_SRCS + CUSTOM_OPS_SRCS + LLAMA_RUNNER_SRCS + ) + set(BUILD_VARIABLES_VARNAMES + _executorch__srcs + _executorch_core__srcs + _portable_kernels__srcs + _optimized_kernels__srcs + _quantized_kernels__srcs + _program_schema__srcs + _optimized_cpublas__srcs + _optimized_native_cpu_ops_oss__srcs + _extension_data_loader__srcs + _extension_module__srcs + _extension_runner_util__srcs + _extension_llm_runner__srcs + _extension_tensor__srcs + _extension_threadpool__srcs + _extension_training__srcs + _train_xor__srcs + _executor_runner__srcs + _size_test__srcs + _mps_executor_runner__srcs + _mps_backend__srcs + _mps_schema__srcs + _xnn_executor_runner__srcs + _xnnpack_backend__srcs + _xnnpack_schema__srcs + _vulkan_schema__srcs + _custom_ops__srcs + _llama_runner__srcs + ) + foreach(filelist_and_varname IN ZIP_LISTS BUILD_VARIABLES_FILELISTS + BUILD_VARIABLES_VARNAMES + ) + append_filelist( + ${filelist_and_varname_0} + "${filelist_and_varname_1}_from_build_variables" + ) + if(NOT ${filelist_and_varname_1} STREQUAL + ${filelist_and_varname_1}_from_build_variables + ) + message( + FATAL_ERROR + "Buck-generated ${filelist_and_varname_1} does not match hardcoded \ +${filelist_and_varname_0} in build_variables.bzl. Left: \ +${${filelist_and_varname_1}}\n \ +Right: ${${filelist_and_varname_1}_from_build_variables}" + ) + endif() + endforeach() endfunction() From 00ce92771a22f649ab5f6ac77cee1f14e4c0b2c6 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Mon, 10 Feb 2025 11:00:23 -0800 Subject: [PATCH 03/29] Update [ghstack-poisoned] --- .../custom_ops/spinquant/test/CMakeLists.txt | 30 +++++++++++++++++++ test/utils/OSSTestConfig.json | 11 +++++++ 2 files changed, 41 insertions(+) create mode 100644 extension/llm/custom_ops/spinquant/test/CMakeLists.txt diff --git a/extension/llm/custom_ops/spinquant/test/CMakeLists.txt b/extension/llm/custom_ops/spinquant/test/CMakeLists.txt new file mode 100644 index 00000000000..63e8207ed0a --- /dev/null +++ b/extension/llm/custom_ops/spinquant/test/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# @generated by test/utils/generate_gtest_cmakelists.py +# +# This file should be formatted with +# ~~~ +# cmake-format -i CMakeLists.txt +# ~~~ +# It should also be cmake-lint clean. +# + +cmake_minimum_required(VERSION 3.19) + +set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..) + +include(${EXECUTORCH_ROOT}/build/Test.cmake) + +set(_test_srcs + fast_hadamard_transform_test.cpp fast_hadamard_transform_test_impl.cpp + op_fast_hadamard_transform_test.cpp +) + +et_cxx_test( + extension_llm_custom_ops_spinquant_test SOURCES ${_test_srcs} EXTRA_LIBS + custom_ops +) diff --git a/test/utils/OSSTestConfig.json b/test/utils/OSSTestConfig.json index 2229b255401..12fa3c03658 100644 --- a/test/utils/OSSTestConfig.json +++ b/test/utils/OSSTestConfig.json @@ -1,4 +1,15 @@ { "tests": [ + { + "directory": "extension/llm/custom_ops/spinquant/test", + "sources": [ + "fast_hadamard_transform_test.cpp", + "fast_hadamard_transform_test_impl.cpp", + "op_fast_hadamard_transform_test.cpp" + ], + "additional_libs": [ + "custom_ops" + ] + }, { "directory": "extension/data_loader/test", "sources": [ From 6440d3debeec9a99c96f83c49f4b1dfa132053ee Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Mon, 10 Feb 2025 11:00:29 -0800 Subject: [PATCH 04/29] Update [ghstack-poisoned] --- build/cmake_deps.toml | 5 +---- extension/llm/custom_ops/CMakeLists.txt | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/build/cmake_deps.toml b/build/cmake_deps.toml index c7680708688..92ddc7d80c8 100644 --- a/build/cmake_deps.toml +++ b/build/cmake_deps.toml @@ -365,10 +365,7 @@ buck_targets = [ "//extension/llm/custom_ops:custom_ops", ] filters = [ - # Second clause is to pick up fht_neon.c/fht_avx.c from FFHT. TODO: - # remove filters and patch extract_sources.py's Buck query to fetch - # srcs; presumably filters is here to remove .h files. - "(.cpp$)|(fht.*\\.c$)", + ".cpp$", ] excludes = [ "^codegen", diff --git a/extension/llm/custom_ops/CMakeLists.txt b/extension/llm/custom_ops/CMakeLists.txt index 16ca4fff805..3203cd32d57 100644 --- a/extension/llm/custom_ops/CMakeLists.txt +++ b/extension/llm/custom_ops/CMakeLists.txt @@ -45,6 +45,22 @@ list(APPEND custom_ops_libs cpuinfo) list(APPEND custom_ops_libs cpublas) list(APPEND custom_ops_libs eigen_blas) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|armv7)$") + list(APPEND _custom_ops__srcs + "extension/llm/custom_ops/spinquant/third-party/FFHT/fht_neon.c" + ) +elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + list(APPEND _custom_ops__srcs + "extension/llm/custom_ops/spinquant/third-party/FFHT/fht_avx.c" + ) +else() + message( + FATAL_ERROR + "Unsupported CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}. (If \ +32-bit x86, try using fht_avx.c and send a PR if it works!)" + ) +endif() + list(TRANSFORM _custom_ops__srcs PREPEND "${EXECUTORCH_ROOT}/") if(NOT EXECUTORCH_BUILD_XNNPACK) From 630bdd001574309ea50159e0a5f1586c0ca19e3f Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Mon, 10 Feb 2025 11:15:39 -0800 Subject: [PATCH 05/29] Update [ghstack-poisoned] --- build/build_variables.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/build/build_variables.bzl b/build/build_variables.bzl index 75f1792f6fa..c3d631a9b47 100644 --- a/build/build_variables.bzl +++ b/build/build_variables.bzl @@ -476,7 +476,6 @@ CUSTOM_OPS_SRCS = [ "extension/llm/custom_ops/op_sdpa.cpp", "extension/llm/custom_ops/op_update_cache.cpp", "extension/llm/custom_ops/spinquant/fast_hadamard_transform.cpp", - "extension/llm/custom_ops/spinquant/third-party/FFHT/fht_neon.c", "kernels/portable/cpu/util/reduce_util.cpp", ] From 098432fdf33a6b94d62ae9fbe4915221aac10cba Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Mon, 10 Feb 2025 15:20:57 -0800 Subject: [PATCH 06/29] Update [ghstack-poisoned] --- build/build_variables.bzl | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/build/build_variables.bzl b/build/build_variables.bzl index c3d631a9b47..3bd0b8028ef 100644 --- a/build/build_variables.bzl +++ b/build/build_variables.bzl @@ -6,14 +6,12 @@ # file. (Remember that bzl files are not exported via ShipIt by default, so you may also # need to update ExecuTorch's ShipIt config.) -# _executorch__srcs EXECUTORCH_SRCS = [ "kernels/prim_ops/et_copy_index.cpp", "kernels/prim_ops/et_view.cpp", "kernels/prim_ops/register_prim_ops.cpp", ] -# _executorch_core__srcs EXECUTORCH_CORE_SRCS = [ "runtime/backend/interface.cpp", "runtime/core/evalue.cpp", @@ -35,7 +33,6 @@ EXECUTORCH_CORE_SRCS = [ "schema/extended_header.cpp", ] -# _portable_kernels__srcs PORTABLE_KERNELS_SRCS = [ "kernels/portable/cpu/op__empty_dim_order.cpp", "kernels/portable/cpu/op__to_dim_order_copy.cpp", @@ -215,7 +212,6 @@ PORTABLE_KERNELS_SRCS = [ "kernels/portable/cpu/util/upsample_util.cpp", ] -# _optimized_kernels__srcs OPTIMIZED_KERNELS_SRCS = [ "extension/parallel/thread_parallel.cpp", "kernels/optimized/blas/BlasKernel.cpp", @@ -235,7 +231,6 @@ OPTIMIZED_KERNELS_SRCS = [ "kernels/optimized/cpu/op_sub.cpp", ] -# _quantized_kernels__srcs QUANTIZED_KERNELS_SRCS = [ "kernels/quantized/cpu/embeddingxb.cpp", "kernels/quantized/cpu/op_add.cpp", @@ -249,13 +244,11 @@ QUANTIZED_KERNELS_SRCS = [ "kernels/quantized/cpu/op_quantize.cpp", ] -# _program_schema__srcs PROGRAM_SCHEMA_SRCS = [ "schema/program.fbs", "schema/scalar_type.fbs", ] -# _optimized_cpublas__srcs OPTIMIZED_CPUBLAS_SRCS = [ "extension/parallel/thread_parallel.cpp", "extension/threadpool/threadpool.cpp", @@ -264,7 +257,6 @@ OPTIMIZED_CPUBLAS_SRCS = [ "kernels/optimized/blas/CPUBlas.cpp", ] -# _optimized_native_cpu_ops_oss__srcs OPTIMIZED_NATIVE_CPU_OPS_OSS_SRCS = [ "codegen/templates/RegisterCodegenUnboxedKernels.cpp", "codegen/templates/RegisterDispatchKeyCustomOps.cpp", @@ -290,24 +282,20 @@ OPTIMIZED_NATIVE_CPU_OPS_OSS_SRCS = [ "kernels/optimized/cpu/op_sub.cpp", ] -# _extension_data_loader__srcs EXTENSION_DATA_LOADER_SRCS = [ "extension/data_loader/file_data_loader.cpp", "extension/data_loader/mmap_data_loader.cpp", ] -# _extension_module__srcs EXTENSION_MODULE_SRCS = [ "extension/module/module.cpp", ] -# _extension_runner_util__srcs EXTENSION_RUNNER_UTIL_SRCS = [ "extension/runner_util/inputs.cpp", "extension/runner_util/inputs_portable.cpp", ] -# _extension_llm_runner__srcs EXTENSION_LLM_RUNNER_SRCS = [ "extension/data_loader/file_data_loader.cpp", "extension/data_loader/mmap_data_loader.cpp", @@ -318,19 +306,16 @@ EXTENSION_LLM_RUNNER_SRCS = [ "extension/tensor/tensor_ptr_maker.cpp", ] -# _extension_tensor__srcs EXTENSION_TENSOR_SRCS = [ "extension/tensor/tensor_ptr.cpp", "extension/tensor/tensor_ptr_maker.cpp", ] -# _extension_threadpool__srcs EXTENSION_THREADPOOL_SRCS = [ "extension/threadpool/threadpool.cpp", "extension/threadpool/threadpool_guard.cpp", ] -# _extension_training__srcs EXTENSION_TRAINING_SRCS = [ "extension/data_loader/file_data_loader.cpp", "extension/data_loader/mmap_data_loader.cpp", @@ -342,7 +327,6 @@ EXTENSION_TRAINING_SRCS = [ "kernels/prim_ops/register_prim_ops.cpp", ] -# _train_xor__srcs TRAIN_XOR_SRCS = [ "extension/data_loader/file_data_loader.cpp", "extension/data_loader/mmap_data_loader.cpp", @@ -356,7 +340,6 @@ TRAIN_XOR_SRCS = [ "extension/training/optimizer/sgd.cpp", ] -# _executor_runner__srcs EXECUTOR_RUNNER_SRCS = [ "examples/portable/executor_runner/executor_runner.cpp", "extension/data_loader/file_data_loader.cpp", @@ -366,13 +349,11 @@ EXECUTOR_RUNNER_SRCS = [ "runtime/executor/test/test_backend_compiler_lib.cpp", ] -# _size_test__srcs SIZE_TEST_SRCS = [ "extension/data_loader/file_data_loader.cpp", "test/size_test.cpp", ] -# _mps_executor_runner__srcs MPS_EXECUTOR_RUNNER_SRCS = [ "backends/apple/mps/runtime/MPSBackend.mm", "backends/apple/mps/runtime/MPSCompiler.mm", @@ -407,7 +388,6 @@ MPS_EXECUTOR_RUNNER_SRCS = [ "extension/runner_util/inputs_portable.cpp", ] -# _mps_backend__srcs MPS_BACKEND_SRCS = [ "backends/apple/mps/runtime/MPSBackend.mm", "backends/apple/mps/runtime/MPSCompiler.mm", @@ -434,12 +414,10 @@ MPS_BACKEND_SRCS = [ "backends/apple/mps/runtime/operations/UnaryOps.mm", ] -# _mps_schema__srcs MPS_SCHEMA_SRCS = [ "backends/apple/mps/serialization/schema.fbs", ] -# _xnn_executor_runner__srcs XNN_EXECUTOR_RUNNER_SRCS = [ "examples/portable/executor_runner/executor_runner.cpp", "extension/data_loader/file_data_loader.cpp", @@ -448,7 +426,6 @@ XNN_EXECUTOR_RUNNER_SRCS = [ "extension/runner_util/inputs_portable.cpp", ] -# _xnnpack_backend__srcs XNNPACK_BACKEND_SRCS = [ "backends/xnnpack/runtime/XNNCompiler.cpp", "backends/xnnpack/runtime/XNNExecutor.cpp", @@ -459,17 +436,14 @@ XNNPACK_BACKEND_SRCS = [ "extension/threadpool/threadpool_guard.cpp", ] -# _xnnpack_schema__srcs XNNPACK_SCHEMA_SRCS = [ "backends/xnnpack/serialization/runtime_schema.fbs", ] -# _vulkan_schema__srcs VULKAN_SCHEMA_SRCS = [ "backends/vulkan/serialization/schema.fbs", ] -# _custom_ops__srcs CUSTOM_OPS_SRCS = [ "extension/llm/custom_ops/op_fallback.cpp", "extension/llm/custom_ops/op_fast_hadamard_transform.cpp", @@ -479,7 +453,6 @@ CUSTOM_OPS_SRCS = [ "kernels/portable/cpu/util/reduce_util.cpp", ] -# _llama_runner__srcs LLAMA_RUNNER_SRCS = [ "examples/models/llama/runner/runner.cpp", "examples/models/llama/tokenizer/llama_tiktoken.cpp", From 4265c311d0321d3b4c81a11ae6517312a4bb40a7 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 11 Feb 2025 16:28:50 -0800 Subject: [PATCH 07/29] Update [ghstack-poisoned] --- build/build_variables.bzl | 15 +++++++++------ runtime/executor/targets.bzl | 7 ++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/build/build_variables.bzl b/build/build_variables.bzl index 3bd0b8028ef..d9ac0f40e3d 100644 --- a/build/build_variables.bzl +++ b/build/build_variables.bzl @@ -12,17 +12,20 @@ EXECUTORCH_SRCS = [ "kernels/prim_ops/register_prim_ops.cpp", ] -EXECUTORCH_CORE_SRCS = [ +PROGRAM_NO_PRIM_OPS_SRCS = [ + "method.cpp", + "method_meta.cpp", + "program.cpp", + "tensor_parser_exec_aten.cpp", +] + +EXECUTORCH_CORE_SRCS = sorted([ "runtime/backend/interface.cpp", "runtime/core/evalue.cpp", "runtime/core/exec_aten/util/tensor_shape_to_c_string.cpp", "runtime/core/exec_aten/util/tensor_util_portable.cpp", "runtime/core/portable_type/tensor_impl.cpp", "runtime/core/tag.cpp", - "runtime/executor/method.cpp", - "runtime/executor/method_meta.cpp", - "runtime/executor/program.cpp", - "runtime/executor/tensor_parser_exec_aten.cpp", "runtime/executor/tensor_parser_portable.cpp", "runtime/kernel/operator_registry.cpp", "runtime/platform/abort.cpp", @@ -31,7 +34,7 @@ EXECUTORCH_CORE_SRCS = [ "runtime/platform/profiler.cpp", "runtime/platform/runtime.cpp", "schema/extended_header.cpp", -] +] + ["runtime/executor/" + x for x in PROGRAM_NO_PRIM_OPS_SRCS]) PORTABLE_KERNELS_SRCS = [ "kernels/portable/cpu/op__empty_dim_order.cpp", diff --git a/runtime/executor/targets.bzl b/runtime/executor/targets.bzl index 158da5d1087..46734b5c686 100644 --- a/runtime/executor/targets.bzl +++ b/runtime/executor/targets.bzl @@ -1,3 +1,4 @@ +load("@fbsource//xplat/executorch/build:build_variables.bzl", "PROGRAM_NO_PRIM_OPS_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") def _program_preprocessor_flags(): @@ -58,11 +59,7 @@ def define_common_targets(): runtime.cxx_library( name = "program_no_prim_ops" + aten_suffix, - srcs = [ - "method.cpp", - "method_meta.cpp", - "program.cpp", - "tensor_parser_exec_aten.cpp", + srcs = PROGRAM_NO_PRIM_OPS_SRCS + [ "tensor_parser{}.cpp".format(aten_suffix if aten_mode else "_portable"), ], headers = [ From 190bea8183da3fe02ac09b206a13e07b48591072 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 11 Feb 2025 16:29:02 -0800 Subject: [PATCH 08/29] Update [ghstack-poisoned] --- build/build_variables.bzl | 13 ++++++++----- runtime/platform/targets.bzl | 8 ++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/build/build_variables.bzl b/build/build_variables.bzl index d9ac0f40e3d..06da88a3056 100644 --- a/build/build_variables.bzl +++ b/build/build_variables.bzl @@ -19,6 +19,13 @@ PROGRAM_NO_PRIM_OPS_SRCS = [ "tensor_parser_exec_aten.cpp", ] +PLATFORM_SRCS = [ + "abort.cpp", + "log.cpp", + "profiler.cpp", + "runtime.cpp", +] + EXECUTORCH_CORE_SRCS = sorted([ "runtime/backend/interface.cpp", "runtime/core/evalue.cpp", @@ -28,13 +35,9 @@ EXECUTORCH_CORE_SRCS = sorted([ "runtime/core/tag.cpp", "runtime/executor/tensor_parser_portable.cpp", "runtime/kernel/operator_registry.cpp", - "runtime/platform/abort.cpp", "runtime/platform/default/posix.cpp", - "runtime/platform/log.cpp", - "runtime/platform/profiler.cpp", - "runtime/platform/runtime.cpp", "schema/extended_header.cpp", -] + ["runtime/executor/" + x for x in PROGRAM_NO_PRIM_OPS_SRCS]) +] + ["runtime/executor/" + x for x in PROGRAM_NO_PRIM_OPS_SRCS] + ["runtime/platform/" + x for x in PLATFORM_SRCS]) PORTABLE_KERNELS_SRCS = [ "kernels/portable/cpu/op__empty_dim_order.cpp", diff --git a/runtime/platform/targets.bzl b/runtime/platform/targets.bzl index 42bb851e2cf..abb64b9bfd3 100644 --- a/runtime/platform/targets.bzl +++ b/runtime/platform/targets.bzl @@ -1,3 +1,4 @@ +load("@fbsource//xplat/executorch/build:build_variables.bzl", "PLATFORM_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") load(":log.bzl", "get_et_logging_flags") @@ -69,12 +70,7 @@ def define_common_targets(): "profiler.h", "runtime.h", ], - srcs = [ - "abort.cpp", - "log.cpp", - "profiler.cpp", - "runtime.cpp", - ], + srcs = PLATFORM_SRCS, exported_preprocessor_flags = get_profiling_flags() + get_et_logging_flags(), exported_deps = [ "//executorch/runtime/platform:pal_interface", From d4aaa3633305f8e378d661db529830720f85bf42 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Mon, 10 Feb 2025 11:00:23 -0800 Subject: [PATCH 09/29] Update [ghstack-poisoned] --- .../custom_ops/spinquant/test/CMakeLists.txt | 30 +++++++++++++++++++ test/utils/OSSTestConfig.json | 11 +++++++ 2 files changed, 41 insertions(+) create mode 100644 extension/llm/custom_ops/spinquant/test/CMakeLists.txt diff --git a/extension/llm/custom_ops/spinquant/test/CMakeLists.txt b/extension/llm/custom_ops/spinquant/test/CMakeLists.txt new file mode 100644 index 00000000000..63e8207ed0a --- /dev/null +++ b/extension/llm/custom_ops/spinquant/test/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# @generated by test/utils/generate_gtest_cmakelists.py +# +# This file should be formatted with +# ~~~ +# cmake-format -i CMakeLists.txt +# ~~~ +# It should also be cmake-lint clean. +# + +cmake_minimum_required(VERSION 3.19) + +set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..) + +include(${EXECUTORCH_ROOT}/build/Test.cmake) + +set(_test_srcs + fast_hadamard_transform_test.cpp fast_hadamard_transform_test_impl.cpp + op_fast_hadamard_transform_test.cpp +) + +et_cxx_test( + extension_llm_custom_ops_spinquant_test SOURCES ${_test_srcs} EXTRA_LIBS + custom_ops +) diff --git a/test/utils/OSSTestConfig.json b/test/utils/OSSTestConfig.json index 5d53a72f6ae..b3eea132c74 100644 --- a/test/utils/OSSTestConfig.json +++ b/test/utils/OSSTestConfig.json @@ -1,4 +1,15 @@ { "tests": [ + { + "directory": "extension/llm/custom_ops/spinquant/test", + "sources": [ + "fast_hadamard_transform_test.cpp", + "fast_hadamard_transform_test_impl.cpp", + "op_fast_hadamard_transform_test.cpp" + ], + "additional_libs": [ + "custom_ops" + ] + }, { "directory": "extension/data_loader/test", "sources": [ From a826c409861adae71ed99f518e040b603a1b7104 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Wed, 12 Feb 2025 15:34:42 -0800 Subject: [PATCH 10/29] Update [ghstack-poisoned] --- test/run_oss_cpp_tests.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test/run_oss_cpp_tests.sh b/test/run_oss_cpp_tests.sh index 59691fd138d..df042af268c 100755 --- a/test/run_oss_cpp_tests.sh +++ b/test/run_oss_cpp_tests.sh @@ -87,7 +87,6 @@ probe_additional_tests() { # CMakeLists.txt rules, that are buildable using build_and_run_test dirs=( examples/models/llama/tokenizer - extension/llm/custom_ops extension/llm/tokenizer ) From ff128f7cf9589529324f9a39cb4c9510805057aa Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Fri, 21 Feb 2025 14:34:16 -0800 Subject: [PATCH 11/29] Update [ghstack-poisoned] --- build/Utils.cmake | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/build/Utils.cmake b/build/Utils.cmake index 113f4829b86..a2c5ce94571 100644 --- a/build/Utils.cmake +++ b/build/Utils.cmake @@ -329,29 +329,25 @@ function(resolve_python_executable) endfunction() # find_package(Torch CONFIG REQUIRED) replacement for targets that have a -# header-only Torch dependency. Because find_package sets variables in the -# parent scope, we use a macro to preserve this rather than maintaining our own -# list of those variables. -macro(find_package_torch_headers) - # We cannot simply use CMAKE_FIND_ROOT_PATH_BOTH, because that does not - # propagate into TorchConfig.cmake. - foreach(mode_kind IN ITEMS PACKAGE LIBRARY INCLUDE) - set(OLD_CMAKE_FIND_ROOT_PATH_MODE_${mode_kind} - ${CMAKE_FIND_ROOT_PATH_MODE_${mode_kind}} - ) - set(CMAKE_FIND_ROOT_PATH_MODE_${mode_kind} BOTH) - endforeach() - find_package_torch() - foreach(mode_kind IN ITEMS PACKAGE LIBRARY INCLUDE) - set(CMAKE_FIND_ROOT_PATH_MODE_${mode_kind} - ${OLD_CMAKE_FIND_ROOT_PATH_MODE_${mode_kind}} - ) - endforeach() -endmacro() +# header-only Torch dependency. +# +# Unlike find_package(Torch ...), this will only set +# TORCH_INCLUDE_DIRS in the parent scope. In particular, it will NOT +# set any of the following: +# - TORCH_FOUND +# - TORCH_LIBRARY +# - TORCH_CXX_FLAGS +function(find_package_torch_headers) + # We implement this way rather than using find_package so that + # cross-compilation can still use the host's installed copy of + # torch, since the headers should be fine. + get_torch_base_path(TORCH_BASE_PATH) + set(TORCH_INCLUDE_DIRS "${TORCH_BASE_PATH}/include;${TORCH_BASE_PATH}/include/torch/csrc/api/include" PARENT_SCOPE) +endfunction() -# Add the Torch CMake configuration to CMAKE_PREFIX_PATH so that find_package -# can find Torch. -function(add_torch_to_cmake_prefix_path) +# Return the base path to the installed Torch Python library in +# outVar. +function(get_torch_base_path outVar) if(NOT PYTHON_EXECUTABLE) resolve_python_executable() endif() @@ -370,6 +366,13 @@ function(add_torch_to_cmake_prefix_path) message("Output:\n${_tmp_torch_path}") message(FATAL_ERROR "Error:\n${_tmp_torch_path_error}") endif() + set(${outVar} ${_tmp_torch_path} PARENT_SCOPE) +endfunction() + +# Add the Torch CMake configuration to CMAKE_PREFIX_PATH so that find_package +# can find Torch. +function(add_torch_to_cmake_prefix_path) + get_torch_base_path(_tmp_torch_path) list(APPEND CMAKE_PREFIX_PATH "${_tmp_torch_path}") set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH}" From 9c7b3d13f576ae6191f158ec27fb3c25334de991 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Mon, 24 Feb 2025 21:14:04 -0800 Subject: [PATCH 12/29] Update [ghstack-poisoned] --- build/build_android_llm_demo.sh | 6 ++++-- extension/android_test/setup.sh | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/build/build_android_llm_demo.sh b/build/build_android_llm_demo.sh index beee2250062..cb2d47fdeb3 100644 --- a/build/build_android_llm_demo.sh +++ b/build/build_android_llm_demo.sh @@ -40,10 +40,11 @@ build_android_native_library() { EXECUTORCH_BUILD_NEURON=OFF fi - cmake --trace . -DCMAKE_INSTALL_PREFIX="${CMAKE_OUT}" \ + cmake . -DCMAKE_INSTALL_PREFIX="${CMAKE_OUT}" \ -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \ -DANDROID_ABI="${ANDROID_ABI}" \ -DANDROID_PLATFORM=android-26 \ + -DBUILD_TESTING=OFF \ -DEXECUTORCH_ENABLE_LOGGING=ON \ -DEXECUTORCH_LOG_LEVEL=Info \ -DEXECUTORCH_BUILD_XNNPACK=ON \ @@ -69,10 +70,11 @@ build_android_native_library() { fi cmake --build "${CMAKE_OUT}" -j "${CMAKE_JOBS}" --target install --config "${EXECUTORCH_CMAKE_BUILD_TYPE}" - cmake --trace extension/android \ + cmake extension/android \ -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \ -DANDROID_ABI="${ANDROID_ABI}" \ -DANDROID_PLATFORM=android-26 \ + -DBUILD_TESTING=OFF \ -DCMAKE_INSTALL_PREFIX="${CMAKE_OUT}" \ -DEXECUTORCH_ENABLE_LOGGING=ON \ -DEXECUTORCH_LOG_LEVEL=Info \ diff --git a/extension/android_test/setup.sh b/extension/android_test/setup.sh index 88c7cf8be84..c21d2c09623 100755 --- a/extension/android_test/setup.sh +++ b/extension/android_test/setup.sh @@ -18,9 +18,10 @@ build_native_library() { CMAKE_OUT="cmake-out-android-${ANDROID_ABI}" ANDROID_NDK="${ANDROID_NDK:-/opt/ndk}" EXECUTORCH_CMAKE_BUILD_TYPE="${EXECUTORCH_CMAKE_BUILD_TYPE:-Release}" - cmake --trace . -DCMAKE_INSTALL_PREFIX="${CMAKE_OUT}" \ + cmake . -DCMAKE_INSTALL_PREFIX="${CMAKE_OUT}" \ -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \ -DANDROID_ABI="${ANDROID_ABI}" \ + -DBUILD_TESTING=OFF \ -DEXECUTORCH_BUILD_XNNPACK=ON \ -DEXECUTORCH_XNNPACK_SHARED_WORKSPACE=ON \ -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ @@ -33,9 +34,10 @@ build_native_library() { cmake --build "${CMAKE_OUT}" -j16 --target install - cmake --trace extension/android \ + cmake extension/android \ -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}"/build/cmake/android.toolchain.cmake \ -DANDROID_ABI="${ANDROID_ABI}" \ + -DBUILD_TESTING=OFF \ -DCMAKE_INSTALL_PREFIX=c"${CMAKE_OUT}" \ -DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \ -DEXECUTORCH_BUILD_LLAMA_JNI=ON \ From 4184f0733faa02702986fe822714c046266b7291 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 12 Aug 2025 19:08:06 -0700 Subject: [PATCH 13/29] Update [ghstack-poisoned] --- .../executorch/build/build_variables.bzl | 36 ++++++++++++++++++- tools/cmake/Codegen.cmake | 19 +++++++--- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/shim_et/xplat/executorch/build/build_variables.bzl b/shim_et/xplat/executorch/build/build_variables.bzl index 5b07688c4c4..3dcbc09de6f 100644 --- a/shim_et/xplat/executorch/build/build_variables.bzl +++ b/shim_et/xplat/executorch/build/build_variables.bzl @@ -15,7 +15,7 @@ # unnecessary duplication, and people will invariably forget to update one or the other. # 3) List them somewhere CMake and Buck can both get at them; that's this file. Buck # files can load() it, and our CMake build evaluates it with Python. (See -# append_filelist in build/Codegen.cmake.) +# executorch_append_filelist in build/Codegen.cmake.) # # Inconveniently, the Buck target layout is much more granular than the CMake library # layout, leading to several complications: @@ -227,6 +227,27 @@ PORTABLE_KERNELS_SRCS = [ "kernels/portable/cpu/pattern/unary_ufunc_realhbf16.cpp", ] +KERNELS_UTIL_ALL_DEPS_SRCS = [ + "kernels/portable/cpu/util/activation_ops_util.cpp", + "kernels/portable/cpu/util/advanced_index_util.cpp", + "kernels/portable/cpu/util/arange_util.cpp", + "kernels/portable/cpu/util/broadcast_util.cpp", + "kernels/portable/cpu/util/copy_ops_util.cpp", + "kernels/portable/cpu/util/delinearize_index.cpp", + "kernels/portable/cpu/util/distance_util.cpp", + "kernels/portable/cpu/util/dtype_util.cpp", + "kernels/portable/cpu/util/index_util.cpp", + "kernels/portable/cpu/util/kernel_ops_util.cpp", + "kernels/portable/cpu/util/matmul_ops_util.cpp", + "kernels/portable/cpu/util/normalization_ops_util.cpp", + "kernels/portable/cpu/util/padding_util.cpp", + "kernels/portable/cpu/util/reduce_util.cpp", + "kernels/portable/cpu/util/repeat_util.cpp", + "kernels/portable/cpu/util/select_copy_util.cpp", + "kernels/portable/cpu/util/slice_util.cpp", + "kernels/portable/cpu/util/upsample_util.cpp", +] + OPTIMIZED_KERNELS_SRCS = [ "kernels/optimized/cpu/binary_ops.cpp", "kernels/optimized/cpu/op_add.cpp", @@ -294,11 +315,24 @@ OPTIMIZED_NATIVE_CPU_OPS_SRCS = [ "kernels/optimized/cpu/op_where.cpp", ] +TEST_BACKEND_COMPILER_LIB_SRCS = [ + "runtime/executor/test/test_backend_compiler_lib.cpp", +] + EXTENSION_DATA_LOADER_SRCS = [ "extension/data_loader/file_data_loader.cpp", "extension/data_loader/mmap_data_loader.cpp", ] +EXTENSION_EVALUE_UTIL_SRCS = [ + "extension/evalue_util/print_evalue.cpp", +] + +EXTENSION_FLAT_TENSOR_SRCS = [ + "extension/flat_tensor/flat_tensor_data_map.cpp", + "extension/flat_tensor/serialize/flat_tensor_header.cpp", +] + EXTENSION_MODULE_SRCS = [ "extension/module/module.cpp", ] diff --git a/tools/cmake/Codegen.cmake b/tools/cmake/Codegen.cmake index b570f0e1e2a..2a40fce8279 100644 --- a/tools/cmake/Codegen.cmake +++ b/tools/cmake/Codegen.cmake @@ -346,7 +346,7 @@ endfunction() # Append the file list in the variable named `name` in build/build_variables.bzl # to the variable named `outputvar` in the caller's scope. -function(append_filelist name outputvar) +function(executorch_append_filelist name outputvar) # configure_file adds its input to the list of CMAKE_RERUN dependencies configure_file( ${PROJECT_SOURCE_DIR}/shim_et/xplat/executorch/build/build_variables.bzl @@ -390,12 +390,16 @@ function(executorch_validate_build_variables) EXECUTORCH_SRCS EXECUTORCH_CORE_SRCS PORTABLE_KERNELS_SRCS + KERNELS_UTIL_ALL_DEPS_SRCS OPTIMIZED_KERNELS_SRCS QUANTIZED_KERNELS_SRCS PROGRAM_SCHEMA_SRCS OPTIMIZED_CPUBLAS_SRCS OPTIMIZED_NATIVE_CPU_OPS_SRCS + TEST_BACKEND_COMPILER_LIB_SRCS EXTENSION_DATA_LOADER_SRCS + EXTENSION_EVALUE_UTIL_SRCS + EXTENSION_FLAT_TENSOR_SRCS EXTENSION_MODULE_SRCS EXTENSION_RUNNER_UTIL_SRCS EXTENSION_LLM_RUNNER_SRCS @@ -419,12 +423,16 @@ function(executorch_validate_build_variables) _executorch__srcs _executorch_core__srcs _portable_kernels__srcs + _kernels_util_all_deps__srcs _optimized_kernels__srcs _quantized_kernels__srcs _program_schema__srcs _optimized_cpublas__srcs _optimized_native_cpu_ops__srcs + _test_backend_compiler_lib__srcs _extension_data_loader__srcs + _extension_evalue_util__srcs + _extension_flat_tensor__srcs _extension_module__srcs _extension_runner_util__srcs _extension_llm_runner__srcs @@ -450,7 +458,7 @@ function(executorch_validate_build_variables) if("${filelist_and_varname_1}" STREQUAL "_custom_ops__srcs") continue() endif() - append_filelist( + executorch_append_filelist( ${filelist_and_varname_0} "${filelist_and_varname_1}_from_build_variables" ) @@ -458,13 +466,14 @@ function(executorch_validate_build_variables) ${filelist_and_varname_1}_from_build_variables ) list(JOIN ${filelist_and_varname_1} "\n" pretty_buck_generated_list) - list(JOIN ${filelist_and_varname_1}_from_build_variables "\n" pretty_hardcoded_list) + list(JOIN ${filelist_and_varname_1}_from_build_variables "\n" + pretty_hardcoded_list + ) message( FATAL_ERROR "Buck-generated ${filelist_and_varname_1} does not match hardcoded " "${filelist_and_varname_0} in build_variables.bzl. Left: " - "${pretty_buck_generated_list}\n " - "Right: ${pretty_hardcoded_list}" + "${pretty_buck_generated_list}\n " "Right: ${pretty_hardcoded_list}" ) endif() endforeach() From 4d2bfd2cd4e60fe5c95063a9bb6e869684f71970 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 12 Aug 2025 19:08:07 -0700 Subject: [PATCH 14/29] Update [ghstack-poisoned] --- kernels/portable/cpu/pattern/targets.bzl | 7 ++----- shim_et/xplat/executorch/build/build_variables.bzl | 11 +++++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kernels/portable/cpu/pattern/targets.bzl b/kernels/portable/cpu/pattern/targets.bzl index 9d6f1a6885d..636c5d2127b 100644 --- a/kernels/portable/cpu/pattern/targets.bzl +++ b/kernels/portable/cpu/pattern/targets.bzl @@ -1,3 +1,4 @@ +load("@fbsource//xplat/executorch/build:build_variables.bzl", "PATTERN_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") def define_common_targets(): @@ -49,11 +50,7 @@ def define_common_targets(): runtime.cxx_library( name = "pattern", - srcs = [ - "unary_ufunc_realhbbf16_to_bool.cpp", - "unary_ufunc_realhbbf16_to_floathbf16.cpp", - "unary_ufunc_realhbf16.cpp", - ], + srcs = PATTERN_SRCS, exported_headers = [ "pattern.h", ], diff --git a/shim_et/xplat/executorch/build/build_variables.bzl b/shim_et/xplat/executorch/build/build_variables.bzl index f78b437a257..4da57fda111 100644 --- a/shim_et/xplat/executorch/build/build_variables.bzl +++ b/shim_et/xplat/executorch/build/build_variables.bzl @@ -62,6 +62,12 @@ EXECUTORCH_CORE_SRCS = sorted([ "schema/extended_header.cpp", ] + ["runtime/executor/" + x for x in PROGRAM_NO_PRIM_OPS_SRCS] + ["runtime/platform/" + x for x in PLATFORM_SRCS]) +PATTERN_SRCS = [ + "unary_ufunc_realhbbf16_to_bool.cpp", + "unary_ufunc_realhbbf16_to_floathbf16.cpp", + "unary_ufunc_realhbf16.cpp", +] + PORTABLE_KERNELS_SRCS = [ "kernels/portable/cpu/op__clone_dim_order.cpp", "kernels/portable/cpu/op__empty_dim_order.cpp", @@ -228,10 +234,7 @@ PORTABLE_KERNELS_SRCS = [ "kernels/portable/cpu/op_view_copy.cpp", "kernels/portable/cpu/op_where.cpp", "kernels/portable/cpu/op_zeros.cpp", - "kernels/portable/cpu/pattern/unary_ufunc_realhbbf16_to_bool.cpp", - "kernels/portable/cpu/pattern/unary_ufunc_realhbbf16_to_floathbf16.cpp", - "kernels/portable/cpu/pattern/unary_ufunc_realhbf16.cpp", -] +] + ["kernels/portable/cpu/pattern/" + x for x in PATTERN_SRCS] KERNELS_UTIL_ALL_DEPS_SRCS = [ "kernels/portable/cpu/util/activation_ops_util.cpp", From d4f9a6f897cdf96245718124ee171e8b0f723293 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 12 Aug 2025 19:08:11 -0700 Subject: [PATCH 15/29] Update [ghstack-poisoned] --- extension/threadpool/targets.bzl | 9 ++++----- shim_et/xplat/executorch/build/build_variables.bzl | 10 ++++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/extension/threadpool/targets.bzl b/extension/threadpool/targets.bzl index 5e7cf2c7dae..6ef55c42434 100644 --- a/extension/threadpool/targets.bzl +++ b/extension/threadpool/targets.bzl @@ -1,4 +1,5 @@ load("@fbsource//xplat/executorch/backends/xnnpack/third-party:third_party_libs.bzl", "third_party_dep") +load("@fbsource//xplat/executorch/build:build_variables.bzl", "THREADPOOL_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") def define_common_targets(): @@ -8,11 +9,9 @@ def define_common_targets(): TARGETS and BUCK files that call this function. """ - _THREADPOOL_SRCS = [ - "thread_parallel.cpp", - "threadpool.cpp", - "threadpool_guard.cpp", - ] + (["fb/threadpool_use_n_threads.cpp"] if not runtime.is_oss else []) + _THREADPOOL_SRCS = THREADPOOL_SRCS + ( + ["fb/threadpool_use_n_threads.cpp"] if not runtime.is_oss else [] + ) _THREADPOOL_HEADERS = [ "threadpool.h", diff --git a/shim_et/xplat/executorch/build/build_variables.bzl b/shim_et/xplat/executorch/build/build_variables.bzl index 4da57fda111..25cba87049f 100644 --- a/shim_et/xplat/executorch/build/build_variables.bzl +++ b/shim_et/xplat/executorch/build/build_variables.bzl @@ -364,12 +364,14 @@ EXTENSION_TENSOR_SRCS = [ "extension/tensor/tensor_ptr_maker.cpp", ] -EXTENSION_THREADPOOL_SRCS = [ - "extension/threadpool/thread_parallel.cpp", - "extension/threadpool/threadpool.cpp", - "extension/threadpool/threadpool_guard.cpp", +THREADPOOL_SRCS = [ + "thread_parallel.cpp", + "threadpool.cpp", + "threadpool_guard.cpp", ] +EXTENSION_THREADPOOL_SRCS = ["extension/threadpool/" + x for x in THREADPOOL_SRCS] + EXTENSION_TRAINING_SRCS = [ "extension/data_loader/file_data_loader.cpp", "extension/data_loader/mmap_data_loader.cpp", From 3ccf05967d44e3aa999ae90b5c0974dc2fc81ec5 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 12 Aug 2025 19:08:17 -0700 Subject: [PATCH 16/29] Update [ghstack-poisoned] --- backends/apple/mps/targets.bzl | 6 +-- .../executorch/build/build_variables.bzl | 52 ++++++++++--------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/backends/apple/mps/targets.bzl b/backends/apple/mps/targets.bzl index 74d79448362..99c97d2b318 100644 --- a/backends/apple/mps/targets.bzl +++ b/backends/apple/mps/targets.bzl @@ -3,6 +3,7 @@ # Provided subject to the LICENSE file in the top level directory. # +load("@fbsource//xplat/executorch/build:build_variables.bzl", "MPS_BACKEND_BUCK_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") def define_common_targets(is_xplat = False, platforms = []): @@ -37,10 +38,7 @@ def define_common_targets(is_xplat = False, platforms = []): "runtime/*.h", "runtime/operations/*.h", ]), - "srcs": native.glob([ - "runtime/*.mm", - "runtime/operations/*.mm", - ]), + "srcs": MPS_BACKEND_BUCK_SRCS, "visibility": [ "//executorch/backends/apple/...", "//executorch/examples/...", diff --git a/shim_et/xplat/executorch/build/build_variables.bzl b/shim_et/xplat/executorch/build/build_variables.bzl index 25cba87049f..f8653036629 100644 --- a/shim_et/xplat/executorch/build/build_variables.bzl +++ b/shim_et/xplat/executorch/build/build_variables.bzl @@ -438,31 +438,33 @@ MPS_EXECUTOR_RUNNER_SRCS = [ "extension/data_loader/file_data_loader.cpp", ] -MPS_BACKEND_SRCS = [ - "backends/apple/mps/runtime/MPSBackend.mm", - "backends/apple/mps/runtime/MPSCompiler.mm", - "backends/apple/mps/runtime/MPSDelegateHeader.mm", - "backends/apple/mps/runtime/MPSDevice.mm", - "backends/apple/mps/runtime/MPSExecutor.mm", - "backends/apple/mps/runtime/MPSGraphBuilder.mm", - "backends/apple/mps/runtime/MPSStream.mm", - "backends/apple/mps/runtime/operations/ActivationOps.mm", - "backends/apple/mps/runtime/operations/BinaryOps.mm", - "backends/apple/mps/runtime/operations/ClampOps.mm", - "backends/apple/mps/runtime/operations/ConstantOps.mm", - "backends/apple/mps/runtime/operations/ConvolutionOps.mm", - "backends/apple/mps/runtime/operations/IndexingOps.mm", - "backends/apple/mps/runtime/operations/LinearAlgebra.mm", - "backends/apple/mps/runtime/operations/NormalizationOps.mm", - "backends/apple/mps/runtime/operations/OperationUtils.mm", - "backends/apple/mps/runtime/operations/PadOps.mm", - "backends/apple/mps/runtime/operations/PoolingOps.mm", - "backends/apple/mps/runtime/operations/QuantDequant.mm", - "backends/apple/mps/runtime/operations/RangeOps.mm", - "backends/apple/mps/runtime/operations/ReduceOps.mm", - "backends/apple/mps/runtime/operations/ShapeOps.mm", - "backends/apple/mps/runtime/operations/UnaryOps.mm", -] +MPS_BACKEND_BUCK_SRCS = [ + "runtime/MPSBackend.mm", + "runtime/MPSCompiler.mm", + "runtime/MPSDelegateHeader.mm", + "runtime/MPSDevice.mm", + "runtime/MPSExecutor.mm", + "runtime/MPSGraphBuilder.mm", + "runtime/MPSStream.mm", + "runtime/operations/ActivationOps.mm", + "runtime/operations/BinaryOps.mm", + "runtime/operations/ClampOps.mm", + "runtime/operations/ConstantOps.mm", + "runtime/operations/ConvolutionOps.mm", + "runtime/operations/IndexingOps.mm", + "runtime/operations/LinearAlgebra.mm", + "runtime/operations/NormalizationOps.mm", + "runtime/operations/OperationUtils.mm", + "runtime/operations/PadOps.mm", + "runtime/operations/PoolingOps.mm", + "runtime/operations/QuantDequant.mm", + "runtime/operations/RangeOps.mm", + "runtime/operations/ReduceOps.mm", + "runtime/operations/ShapeOps.mm", + "runtime/operations/UnaryOps.mm", +] + +MPS_BACKEND_SRCS = ["backends/apple/mps/" + x for x in MPS_BACKEND_BUCK_SRCS] MPS_SCHEMA_SRCS = [ "backends/apple/mps/serialization/schema.fbs", From c24a61929a3e63969fd31a53c8fb1358eeb7e165 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 12 Aug 2025 19:08:22 -0700 Subject: [PATCH 17/29] Update [ghstack-poisoned] --- backends/xnnpack/targets.bzl | 6 ++---- .../xplat/executorch/build/build_variables.bzl | 16 +++++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/backends/xnnpack/targets.bzl b/backends/xnnpack/targets.bzl index aee5104b17a..0eab89a00f9 100644 --- a/backends/xnnpack/targets.bzl +++ b/backends/xnnpack/targets.bzl @@ -1,4 +1,5 @@ load("@fbsource//xplat/executorch/backends/xnnpack/third-party:third_party_libs.bzl", "third_party_dep") +load("@fbsource//xplat/executorch/build:build_variables.bzl", "XNNPACK_BACKEND_BUCK_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_aten_mode_options", "runtime") def _get_preprocessor_flags(): @@ -37,10 +38,7 @@ def define_common_targets(): aten_suffix = "_aten" if aten_mode else "" runtime.cxx_library( name = "xnnpack_backend" + aten_suffix, - srcs = native.glob([ - "runtime/*.cpp", - "runtime/profiling/*.cpp", - ]), + srcs = XNNPACK_BACKEND_BUCK_SRCS, headers = native.glob([ "runtime/*.h", "runtime/profiling/*.h", diff --git a/shim_et/xplat/executorch/build/build_variables.bzl b/shim_et/xplat/executorch/build/build_variables.bzl index f8653036629..9549e866dab 100644 --- a/shim_et/xplat/executorch/build/build_variables.bzl +++ b/shim_et/xplat/executorch/build/build_variables.bzl @@ -475,15 +475,17 @@ XNN_EXECUTOR_RUNNER_SRCS = [ "extension/data_loader/file_data_loader.cpp", ] -XNNPACK_BACKEND_SRCS = [ - "backends/xnnpack/runtime/XNNCompiler.cpp", - "backends/xnnpack/runtime/XNNExecutor.cpp", - "backends/xnnpack/runtime/XNNHeader.cpp", - "backends/xnnpack/runtime/XNNPACKBackend.cpp", - "backends/xnnpack/runtime/XNNWeightsCache.cpp", - "backends/xnnpack/runtime/profiling/XNNProfiler.cpp", +XNNPACK_BACKEND_BUCK_SRCS = [ + "runtime/XNNCompiler.cpp", + "runtime/XNNExecutor.cpp", + "runtime/XNNHeader.cpp", + "runtime/XNNPACKBackend.cpp", + "runtime/XNNWeightsCache.cpp", + "runtime/profiling/XNNProfiler.cpp", ] +XNNPACK_BACKEND_SRCS = ["backends/xnnpack/" + x for x in XNNPACK_BACKEND_BUCK_SRCS] + XNNPACK_SCHEMA_SRCS = [ "backends/xnnpack/serialization/runtime_schema.fbs", ] From 69ed8e006e50ed9d892fde98dd0080c0991763e0 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 12 Aug 2025 19:08:26 -0700 Subject: [PATCH 18/29] Update [ghstack-poisoned] --- extension/llm/custom_ops/targets.bzl | 8 ++------ shim_et/xplat/executorch/build/build_variables.bzl | 14 ++++++++------ tools/cmake/Codegen.cmake | 3 --- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/extension/llm/custom_ops/targets.bzl b/extension/llm/custom_ops/targets.bzl index 545f6516bb7..26198ec0854 100644 --- a/extension/llm/custom_ops/targets.bzl +++ b/extension/llm/custom_ops/targets.bzl @@ -1,3 +1,4 @@ +load("@fbsource//xplat/executorch/build:build_variables.bzl", "EXTENSION_LLM_CUSTOM_OPS_BUCK_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") load( "@fbsource//xplat/executorch/kernels/optimized:lib_defs.bzl", @@ -30,12 +31,7 @@ def define_common_targets(): for mkl_dep in ["", "_mkl_noomp"]: runtime.cxx_library( name = "custom_ops" + mkl_dep, - srcs = [ - "op_fallback.cpp", - "op_fast_hadamard_transform.cpp", - "op_sdpa.cpp", - "op_update_cache.cpp", - ], + srcs = EXTENSION_LLM_CUSTOM_OPS_BUCK_SRCS, exported_headers = [ "op_fallback.h", "op_fast_hadamard_transform.h", diff --git a/shim_et/xplat/executorch/build/build_variables.bzl b/shim_et/xplat/executorch/build/build_variables.bzl index 9549e866dab..ba3452dbc11 100644 --- a/shim_et/xplat/executorch/build/build_variables.bzl +++ b/shim_et/xplat/executorch/build/build_variables.bzl @@ -494,13 +494,15 @@ VULKAN_SCHEMA_SRCS = [ "backends/vulkan/serialization/schema.fbs", ] -CUSTOM_OPS_SRCS = [ - "extension/llm/custom_ops/op_fallback.cpp", - "extension/llm/custom_ops/op_fast_hadamard_transform.cpp", - "extension/llm/custom_ops/op_sdpa.cpp", - "extension/llm/custom_ops/op_update_cache.cpp", +EXTENSION_LLM_CUSTOM_OPS_BUCK_SRCS = [ + "op_fallback.cpp", + "op_fast_hadamard_transform.cpp", + "op_sdpa.cpp", + "op_update_cache.cpp", +] + +CUSTOM_OPS_SRCS = ["extension/llm/custom_ops/" + x for x in EXTENSION_LLM_CUSTOM_OPS_BUCK_SRCS] + [ "extension/llm/custom_ops/spinquant/fast_hadamard_transform.cpp", - "kernels/portable/cpu/util/reduce_util.cpp", ] LLAMA_RUNNER_SRCS = [ diff --git a/tools/cmake/Codegen.cmake b/tools/cmake/Codegen.cmake index 2a40fce8279..3557979796b 100644 --- a/tools/cmake/Codegen.cmake +++ b/tools/cmake/Codegen.cmake @@ -455,9 +455,6 @@ function(executorch_validate_build_variables) foreach(filelist_and_varname IN ZIP_LISTS BUILD_VARIABLES_FILELISTS BUILD_VARIABLES_VARNAMES ) - if("${filelist_and_varname_1}" STREQUAL "_custom_ops__srcs") - continue() - endif() executorch_append_filelist( ${filelist_and_varname_0} "${filelist_and_varname_1}_from_build_variables" From 850e28841ce4b534b57e92be80ececab99935eb9 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 12 Aug 2025 19:08:31 -0700 Subject: [PATCH 19/29] Update [ghstack-poisoned] --- CMakeLists.txt | 9 +-- tools/cmake/Codegen.cmake | 152 +++++++++++++++++++++----------------- 2 files changed, 88 insertions(+), 73 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a12eef1a9ea..eb523792f7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -334,7 +334,10 @@ set(_common_include_directories # # The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. # - +if(EXECUTORCH_SRCS_FILE) + message(WARNING "EXECUTORCH_SRCS_FILE is no longer necessary and will not affect the build.") +endif() +executorch_load_build_variables() if(NOT EXECUTORCH_SRCS_FILE) # A file wasn't provided. Run a script to extract the source lists from the # buck2 build system and write them to a file we can include. @@ -347,10 +350,6 @@ if(NOT EXECUTORCH_SRCS_FILE) executorch_validate_build_variables() endif() -# This file defines the `___srcs` variables used below. -message(STATUS "executorch: Using sources file ${EXECUTORCH_SRCS_FILE}") -include(${EXECUTORCH_SRCS_FILE}) - # Detect if an iOS toolchain is set. if(CMAKE_TOOLCHAIN_FILE MATCHES ".*(iOS|ios\.toolchain)\.cmake$") set(CMAKE_TOOLCHAIN_IOS ON) diff --git a/tools/cmake/Codegen.cmake b/tools/cmake/Codegen.cmake index 3557979796b..461e6842a8f 100644 --- a/tools/cmake/Codegen.cmake +++ b/tools/cmake/Codegen.cmake @@ -375,6 +375,73 @@ function(executorch_append_filelist name outputvar) ) endfunction() +set(EXECUTORCH_BUILD_VARIABLES_FILELISTS + EXECUTORCH_SRCS + EXECUTORCH_CORE_SRCS + PORTABLE_KERNELS_SRCS + KERNELS_UTIL_ALL_DEPS_SRCS + OPTIMIZED_KERNELS_SRCS + QUANTIZED_KERNELS_SRCS + PROGRAM_SCHEMA_SRCS + OPTIMIZED_CPUBLAS_SRCS + OPTIMIZED_NATIVE_CPU_OPS_SRCS + TEST_BACKEND_COMPILER_LIB_SRCS + EXTENSION_DATA_LOADER_SRCS + EXTENSION_EVALUE_UTIL_SRCS + EXTENSION_FLAT_TENSOR_SRCS + EXTENSION_MODULE_SRCS + EXTENSION_RUNNER_UTIL_SRCS + EXTENSION_LLM_RUNNER_SRCS + EXTENSION_TENSOR_SRCS + EXTENSION_THREADPOOL_SRCS + EXTENSION_TRAINING_SRCS + TRAIN_XOR_SRCS + EXECUTOR_RUNNER_SRCS + SIZE_TEST_SRCS + MPS_EXECUTOR_RUNNER_SRCS + MPS_BACKEND_SRCS + MPS_SCHEMA_SRCS + XNN_EXECUTOR_RUNNER_SRCS + XNNPACK_BACKEND_SRCS + XNNPACK_SCHEMA_SRCS + VULKAN_SCHEMA_SRCS + CUSTOM_OPS_SRCS + LLAMA_RUNNER_SRCS +) +set(EXECUTORCH_BUILD_VARIABLES_VARNAMES + _executorch__srcs + _executorch_core__srcs + _portable_kernels__srcs + _kernels_util_all_deps__srcs + _optimized_kernels__srcs + _quantized_kernels__srcs + _program_schema__srcs + _optimized_cpublas__srcs + _optimized_native_cpu_ops__srcs + _test_backend_compiler_lib__srcs + _extension_data_loader__srcs + _extension_evalue_util__srcs + _extension_flat_tensor__srcs + _extension_module__srcs + _extension_runner_util__srcs + _extension_llm_runner__srcs + _extension_tensor__srcs + _extension_threadpool__srcs + _extension_training__srcs + _train_xor__srcs + _executor_runner__srcs + _size_test__srcs + _mps_executor_runner__srcs + _mps_backend__srcs + _mps_schema__srcs + _xnn_executor_runner__srcs + _xnnpack_backend__srcs + _xnnpack_schema__srcs + _vulkan_schema__srcs + _custom_ops__srcs + _llama_runner__srcs +) + # Fail the build if the src lists in build_variables.bzl do not match the src # lists extracted from Buck and placed into EXECUTORCH_SRCS_FILE. This is # intended to be a safety mechanism while we are in the process of removing Buck @@ -386,74 +453,8 @@ endfunction() # involve getting these lists to match! function(executorch_validate_build_variables) include(${EXECUTORCH_SRCS_FILE}) - set(BUILD_VARIABLES_FILELISTS - EXECUTORCH_SRCS - EXECUTORCH_CORE_SRCS - PORTABLE_KERNELS_SRCS - KERNELS_UTIL_ALL_DEPS_SRCS - OPTIMIZED_KERNELS_SRCS - QUANTIZED_KERNELS_SRCS - PROGRAM_SCHEMA_SRCS - OPTIMIZED_CPUBLAS_SRCS - OPTIMIZED_NATIVE_CPU_OPS_SRCS - TEST_BACKEND_COMPILER_LIB_SRCS - EXTENSION_DATA_LOADER_SRCS - EXTENSION_EVALUE_UTIL_SRCS - EXTENSION_FLAT_TENSOR_SRCS - EXTENSION_MODULE_SRCS - EXTENSION_RUNNER_UTIL_SRCS - EXTENSION_LLM_RUNNER_SRCS - EXTENSION_TENSOR_SRCS - EXTENSION_THREADPOOL_SRCS - EXTENSION_TRAINING_SRCS - TRAIN_XOR_SRCS - EXECUTOR_RUNNER_SRCS - SIZE_TEST_SRCS - MPS_EXECUTOR_RUNNER_SRCS - MPS_BACKEND_SRCS - MPS_SCHEMA_SRCS - XNN_EXECUTOR_RUNNER_SRCS - XNNPACK_BACKEND_SRCS - XNNPACK_SCHEMA_SRCS - VULKAN_SCHEMA_SRCS - CUSTOM_OPS_SRCS - LLAMA_RUNNER_SRCS - ) - set(BUILD_VARIABLES_VARNAMES - _executorch__srcs - _executorch_core__srcs - _portable_kernels__srcs - _kernels_util_all_deps__srcs - _optimized_kernels__srcs - _quantized_kernels__srcs - _program_schema__srcs - _optimized_cpublas__srcs - _optimized_native_cpu_ops__srcs - _test_backend_compiler_lib__srcs - _extension_data_loader__srcs - _extension_evalue_util__srcs - _extension_flat_tensor__srcs - _extension_module__srcs - _extension_runner_util__srcs - _extension_llm_runner__srcs - _extension_tensor__srcs - _extension_threadpool__srcs - _extension_training__srcs - _train_xor__srcs - _executor_runner__srcs - _size_test__srcs - _mps_executor_runner__srcs - _mps_backend__srcs - _mps_schema__srcs - _xnn_executor_runner__srcs - _xnnpack_backend__srcs - _xnnpack_schema__srcs - _vulkan_schema__srcs - _custom_ops__srcs - _llama_runner__srcs - ) - foreach(filelist_and_varname IN ZIP_LISTS BUILD_VARIABLES_FILELISTS - BUILD_VARIABLES_VARNAMES + foreach(filelist_and_varname IN ZIP_LISTS EXECUTORCH_BUILD_VARIABLES_FILELISTS + EXECUTORCH_BUILD_VARIABLES_VARNAMES ) executorch_append_filelist( ${filelist_and_varname_0} @@ -475,3 +476,18 @@ function(executorch_validate_build_variables) endif() endforeach() endfunction() + +function(executorch_load_build_variables) + foreach(filelist_and_varname IN ZIP_LISTS EXECUTORCH_BUILD_VARIABLES_FILELISTS + EXECUTORCH_BUILD_VARIABLES_VARNAMES + ) + executorch_append_filelist( + ${filelist_and_varname_0} + "${filelist_and_varname_1}" + ) + set(${filelist_and_varname_1} + "${${filelist_and_varname_1}}" + PARENT_SCOPE + ) + endforeach() +endfunction() From 97b2cdf60e9b75a53437f56c56e900bc302cf122 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Wed, 13 Aug 2025 15:44:30 -0700 Subject: [PATCH 20/29] Update [ghstack-poisoned] --- examples/apple/mps/CMakeLists.txt | 10 ++-------- examples/mediatek/CMakeLists.txt | 8 ++------ examples/portable/custom_ops/CMakeLists.txt | 10 ++-------- examples/qualcomm/CMakeLists.txt | 8 ++------ examples/selective_build/CMakeLists.txt | 10 ++-------- extension/llm/custom_ops/CMakeLists.txt | 10 ++-------- extension/llm/runner/CMakeLists.txt | 10 ++-------- test/CMakeLists.txt | 9 +++------ tools/cmake/Codegen.cmake | 7 +++++-- 9 files changed, 22 insertions(+), 60 deletions(-) diff --git a/examples/apple/mps/CMakeLists.txt b/examples/apple/mps/CMakeLists.txt index 3f61cedec8e..8a562dd206b 100644 --- a/examples/apple/mps/CMakeLists.txt +++ b/examples/apple/mps/CMakeLists.txt @@ -76,16 +76,10 @@ if(NOT CMAKE_TOOLCHAIN_FILE MATCHES ".*(iOS|ios\.toolchain)\.cmake$") ) # - # The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. + # The `__srcs` lists are defined by executorch_load_build_variables. # - set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../../executorch_srcs.cmake" - ) - - extract_sources(${EXECUTORCH_SRCS_FILE}) - + executorch_load_build_variables() set(_mps_schema_headers ${CMAKE_BINARY_DIR}/../../../schema/include/) - include(${EXECUTORCH_SRCS_FILE}) target_include_directories( bundled_program INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/../../../devtools/include diff --git a/examples/mediatek/CMakeLists.txt b/examples/mediatek/CMakeLists.txt index 57c4b13e5cb..2bd08de2ffb 100644 --- a/examples/mediatek/CMakeLists.txt +++ b/examples/mediatek/CMakeLists.txt @@ -36,13 +36,9 @@ set(_common_include_directories ) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../executorch_srcs.cmake" -) -extract_sources(${EXECUTORCH_SRCS_FILE}) -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() # Find prebuilt libraries. executorch package should contain portable_ops_lib, # etdump, bundled_program. diff --git a/examples/portable/custom_ops/CMakeLists.txt b/examples/portable/custom_ops/CMakeLists.txt index c4a00e47991..4188554af79 100644 --- a/examples/portable/custom_ops/CMakeLists.txt +++ b/examples/portable/custom_ops/CMakeLists.txt @@ -59,15 +59,9 @@ option( # ------------------------------- OPTIONS END -------------------------------- # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../../executorch_srcs.cmake" -) - -extract_sources(${EXECUTORCH_SRCS_FILE}) - -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() # Generate C++ bindings to register kernels into both PyTorch (for AOT) and # Executorch (for runtime). diff --git a/examples/qualcomm/CMakeLists.txt b/examples/qualcomm/CMakeLists.txt index d33d666b9c0..19190b6f794 100644 --- a/examples/qualcomm/CMakeLists.txt +++ b/examples/qualcomm/CMakeLists.txt @@ -35,13 +35,9 @@ find_package(gflags REQUIRED) set(_common_compile_options -Wno-deprecated-declarations -fPIC) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../executorch_srcs.cmake" -) -extract_sources(${EXECUTORCH_SRCS_FILE}) -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() get_filename_component( EXECUTORCH_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE diff --git a/examples/selective_build/CMakeLists.txt b/examples/selective_build/CMakeLists.txt index 3cc5e759ac4..dbff311a39a 100644 --- a/examples/selective_build/CMakeLists.txt +++ b/examples/selective_build/CMakeLists.txt @@ -77,15 +77,9 @@ option(EXECUTORCH_DTYPE_SELECTIVE_BUILD "Enable dtype selective build." OFF) # ------------------------------- OPTIONS END -------------------------------- # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../executorch_srcs.cmake" -) - -extract_sources(${EXECUTORCH_SRCS_FILE}) - -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() # # select_build_lib: C++ library to register selected ops in custom kernel diff --git a/extension/llm/custom_ops/CMakeLists.txt b/extension/llm/custom_ops/CMakeLists.txt index c5eba4b7a19..1678dc80296 100644 --- a/extension/llm/custom_ops/CMakeLists.txt +++ b/extension/llm/custom_ops/CMakeLists.txt @@ -25,15 +25,9 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../../executorch_srcs.cmake" -) - -extract_sources(${EXECUTORCH_SRCS_FILE}) - -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() set(_common_include_directories $ diff --git a/extension/llm/runner/CMakeLists.txt b/extension/llm/runner/CMakeLists.txt index f5933e82e32..c9a72fe61a0 100644 --- a/extension/llm/runner/CMakeLists.txt +++ b/extension/llm/runner/CMakeLists.txt @@ -24,15 +24,9 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../../executorch_srcs.cmake" -) - -extract_sources(${EXECUTORCH_SRCS_FILE}) - -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() # build llm runner library list(TRANSFORM _extension_llm_runner__srcs PREPEND "${EXECUTORCH_ROOT}/") diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5ad429e822f..870da77deb6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -23,6 +23,7 @@ set(CMAKE_CXX_STANDARD 17) set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/..) +include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake) include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) # Find prebuilt executorch library @@ -34,13 +35,9 @@ find_package(executorch CONFIG REQUIRED FIND_ROOT_PATH_BOTH) set(_common_include_directories ${EXECUTORCH_ROOT}/..) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE "${CMAKE_CURRENT_BINARY_DIR}/../executorch_srcs.cmake") - -extract_sources(${EXECUTORCH_SRCS_FILE}) - -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() # Since extract_sources.py is not returning absolute values, we need to patch # the source paths. diff --git a/tools/cmake/Codegen.cmake b/tools/cmake/Codegen.cmake index 30e33cd418e..93ba0f890a8 100644 --- a/tools/cmake/Codegen.cmake +++ b/tools/cmake/Codegen.cmake @@ -349,13 +349,16 @@ endfunction() function(executorch_append_filelist name outputvar) # configure_file adds its input to the list of CMAKE_RERUN dependencies configure_file( - ${PROJECT_SOURCE_DIR}/shim_et/xplat/executorch/build/build_variables.bzl + ${EXECUTORCH_ROOT}/shim_et/xplat/executorch/build/build_variables.bzl ${PROJECT_BINARY_DIR}/build_variables.bzl COPYONLY ) + if(NOT PYTHON_EXECUTABLE) + resolve_python_executable() + endif() execute_process( COMMAND "${PYTHON_EXECUTABLE}" -c - "exec(open('${PROJECT_SOURCE_DIR}/shim_et/xplat/executorch/build/build_variables.bzl').read());print(';'.join(${name}))" + "exec(open('${EXECUTORCH_ROOT}/shim_et/xplat/executorch/build/build_variables.bzl').read());print(';'.join(${name}))" WORKING_DIRECTORY "${_rootdir}" RESULT_VARIABLE _retval OUTPUT_VARIABLE _tempvar From 689fdc383fbd26e183378f5fa667157524ad81eb Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Wed, 13 Aug 2025 15:44:36 -0700 Subject: [PATCH 21/29] Update [ghstack-poisoned] --- CMakeLists.txt | 11 --- tools/cmake/Codegen.cmake | 198 +++++++++++++------------------------- 2 files changed, 66 insertions(+), 143 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 57fc81600f2..bb6eeab4217 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -318,17 +318,6 @@ if(EXECUTORCH_SRCS_FILE) ) endif() executorch_load_build_variables() -if(NOT EXECUTORCH_SRCS_FILE) - # A file wasn't provided. Run a script to extract the source lists from the - # buck2 build system and write them to a file we can include. - # - # NOTE: This will only happen once during cmake setup, so it will not re-run - # if the buck2 targets change. - message(STATUS "executorch: Generating source lists") - set(EXECUTORCH_SRCS_FILE "${CMAKE_CURRENT_BINARY_DIR}/executorch_srcs.cmake") - extract_sources(${EXECUTORCH_SRCS_FILE}) - executorch_validate_build_variables() -endif() # Detect if an iOS toolchain is set. if(CMAKE_TOOLCHAIN_FILE MATCHES ".*(iOS|ios\.toolchain)\.cmake$") diff --git a/tools/cmake/Codegen.cmake b/tools/cmake/Codegen.cmake index 93ba0f890a8..aa9c2133851 100644 --- a/tools/cmake/Codegen.cmake +++ b/tools/cmake/Codegen.cmake @@ -378,139 +378,73 @@ function(executorch_append_filelist name outputvar) ) endfunction() -set(EXECUTORCH_BUILD_VARIABLES_FILELISTS - EXECUTORCH_SRCS - EXECUTORCH_CORE_SRCS - PORTABLE_KERNELS_SRCS - KERNELS_UTIL_ALL_DEPS_SRCS - OPTIMIZED_KERNELS_SRCS - QUANTIZED_KERNELS_SRCS - PROGRAM_SCHEMA_SRCS - OPTIMIZED_CPUBLAS_SRCS - OPTIMIZED_NATIVE_CPU_OPS_SRCS - TEST_BACKEND_COMPILER_LIB_SRCS - EXTENSION_DATA_LOADER_SRCS - EXTENSION_EVALUE_UTIL_SRCS - EXTENSION_FLAT_TENSOR_SRCS - EXTENSION_MODULE_SRCS - EXTENSION_RUNNER_UTIL_SRCS - EXTENSION_LLM_RUNNER_SRCS - EXTENSION_TENSOR_SRCS - EXTENSION_THREADPOOL_SRCS - EXTENSION_TRAINING_SRCS - TRAIN_XOR_SRCS - EXECUTOR_RUNNER_SRCS - SIZE_TEST_SRCS - MPS_EXECUTOR_RUNNER_SRCS - MPS_BACKEND_SRCS - MPS_SCHEMA_SRCS - XNN_EXECUTOR_RUNNER_SRCS - XNNPACK_BACKEND_SRCS - XNNPACK_SCHEMA_SRCS - VULKAN_SCHEMA_SRCS - CUSTOM_OPS_SRCS - LLAMA_RUNNER_SRCS -) -set(EXECUTORCH_BUILD_VARIABLES_VARNAMES - _executorch__srcs - _executorch_core__srcs - _portable_kernels__srcs - _kernels_util_all_deps__srcs - _optimized_kernels__srcs - _quantized_kernels__srcs - _program_schema__srcs - _optimized_cpublas__srcs - _optimized_native_cpu_ops__srcs - _test_backend_compiler_lib__srcs - _extension_data_loader__srcs - _extension_evalue_util__srcs - _extension_flat_tensor__srcs - _extension_module__srcs - _extension_runner_util__srcs - _extension_llm_runner__srcs - _extension_tensor__srcs - _extension_threadpool__srcs - _extension_training__srcs - _train_xor__srcs - _executor_runner__srcs - _size_test__srcs - _mps_executor_runner__srcs - _mps_backend__srcs - _mps_schema__srcs - _xnn_executor_runner__srcs - _xnnpack_backend__srcs - _xnnpack_schema__srcs - _vulkan_schema__srcs - _custom_ops__srcs - _llama_runner__srcs -) - -# Fail the build if the src lists in build_variables.bzl do not match the src -# lists extracted from Buck and placed into EXECUTORCH_SRCS_FILE. This is -# intended to be a safety mechanism while we are in the process of removing Buck -# from the CMake build and replacing it with build_variables.bzl; if you are -# seeing failures after you have intentionally changed Buck srcs, then simply -# update build_variables.bzl. If you are seeing failures after changing -# something about the build system, make sure your changes will work both before -# and after we finish replacing Buck with build_variables.bzl, which should -# involve getting these lists to match! -function(executorch_validate_build_variables) - include(${EXECUTORCH_SRCS_FILE}) - foreach(filelist_and_varname IN - ZIP_LISTS EXECUTORCH_BUILD_VARIABLES_FILELISTS - EXECUTORCH_BUILD_VARIABLES_VARNAMES - ) - executorch_append_filelist( - ${filelist_and_varname_0} - "${filelist_and_varname_1}_from_build_variables" - ) - # The Buck and CMake mechanisms for getting the default PAL set up are - # different. Prevent the Buck choice from flowing into CMake and causing - # validation to fail, just like we do in our CMakeLists.txt. - if("${filelist_and_varname_1}" STREQUAL "_executorch_core__srcs") - list(FILTER ${filelist_and_varname_1} EXCLUDE REGEX - "runtime/platform/default/[^/]*.cpp$" - ) - endif() - if(NOT ${filelist_and_varname_1} STREQUAL - ${filelist_and_varname_1}_from_build_variables - ) - set(generated_items_not_in_build_variables ${${filelist_and_varname_1}}) - list(REMOVE_ITEM generated_items_not_in_build_variables - ${${filelist_and_varname_1}_from_build_variables} - ) - - set(build_variables_items_not_in_generated - ${${filelist_and_varname_1}_from_build_variables} - ) - list(REMOVE_ITEM build_variables_items_not_in_generated - ${${filelist_and_varname_1}} - ) - - list(JOIN generated_items_not_in_build_variables "\n" - pretty_generated_items_not_in_build_variables - ) - list(JOIN build_variables_items_not_in_generated "\n" - pretty_build_variables_items_not_in_generated - ) - if(NOT pretty_generated_items_not_in_build_variables) - set(pretty_generated_items_not_in_build_variables "") - endif() - if(NOT pretty_build_variables_items_not_in_generated) - set(pretty_build_variables_items_not_in_generated "") - endif() - message( - FATAL_ERROR - "Buck-generated ${filelist_and_varname_1} does not match hardcoded " - "${filelist_and_varname_0} in build_variables.bzl. Buck-generated items not in build_variables.bzl: " - "${pretty_generated_items_not_in_build_variables}\n " - "build_variables.bzl items not in buck-generated list: ${pretty_build_variables_items_not_in_generated}" - ) - endif() - endforeach() -endfunction() - function(executorch_load_build_variables) + set(EXECUTORCH_BUILD_VARIABLES_FILELISTS + EXECUTORCH_SRCS + EXECUTORCH_CORE_SRCS + PORTABLE_KERNELS_SRCS + KERNELS_UTIL_ALL_DEPS_SRCS + OPTIMIZED_KERNELS_SRCS + QUANTIZED_KERNELS_SRCS + PROGRAM_SCHEMA_SRCS + OPTIMIZED_CPUBLAS_SRCS + OPTIMIZED_NATIVE_CPU_OPS_SRCS + TEST_BACKEND_COMPILER_LIB_SRCS + EXTENSION_DATA_LOADER_SRCS + EXTENSION_EVALUE_UTIL_SRCS + EXTENSION_FLAT_TENSOR_SRCS + EXTENSION_MODULE_SRCS + EXTENSION_RUNNER_UTIL_SRCS + EXTENSION_LLM_RUNNER_SRCS + EXTENSION_TENSOR_SRCS + EXTENSION_THREADPOOL_SRCS + EXTENSION_TRAINING_SRCS + TRAIN_XOR_SRCS + EXECUTOR_RUNNER_SRCS + SIZE_TEST_SRCS + MPS_EXECUTOR_RUNNER_SRCS + MPS_BACKEND_SRCS + MPS_SCHEMA_SRCS + XNN_EXECUTOR_RUNNER_SRCS + XNNPACK_BACKEND_SRCS + XNNPACK_SCHEMA_SRCS + VULKAN_SCHEMA_SRCS + CUSTOM_OPS_SRCS + LLAMA_RUNNER_SRCS + ) + set(EXECUTORCH_BUILD_VARIABLES_VARNAMES + _executorch__srcs + _executorch_core__srcs + _portable_kernels__srcs + _kernels_util_all_deps__srcs + _optimized_kernels__srcs + _quantized_kernels__srcs + _program_schema__srcs + _optimized_cpublas__srcs + _optimized_native_cpu_ops__srcs + _test_backend_compiler_lib__srcs + _extension_data_loader__srcs + _extension_evalue_util__srcs + _extension_flat_tensor__srcs + _extension_module__srcs + _extension_runner_util__srcs + _extension_llm_runner__srcs + _extension_tensor__srcs + _extension_threadpool__srcs + _extension_training__srcs + _train_xor__srcs + _executor_runner__srcs + _size_test__srcs + _mps_executor_runner__srcs + _mps_backend__srcs + _mps_schema__srcs + _xnn_executor_runner__srcs + _xnnpack_backend__srcs + _xnnpack_schema__srcs + _vulkan_schema__srcs + _custom_ops__srcs + _llama_runner__srcs + ) foreach(filelist_and_varname IN ZIP_LISTS EXECUTORCH_BUILD_VARIABLES_FILELISTS EXECUTORCH_BUILD_VARIABLES_VARNAMES From ef3c565dcd157bb04568ac14da460c9293129c08 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Wed, 13 Aug 2025 15:44:40 -0700 Subject: [PATCH 22/29] Update [ghstack-poisoned] --- CMakeLists.txt | 5 --- tools/cmake/Utils.cmake | 68 ----------------------------------------- 2 files changed, 73 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb6eeab4217..c36a19125f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,11 +79,6 @@ if(NOT PYTHON_EXECUTABLE) endif() announce_configured_options(PYTHON_EXECUTABLE) -if(NOT BUCK2) - resolve_buck2() -endif() -announce_configured_options(BUCK2) - announce_configured_options(CMAKE_CXX_COMPILER_ID) announce_configured_options(CMAKE_TOOLCHAIN_FILE) announce_configured_options(BUILD_TESTING) diff --git a/tools/cmake/Utils.cmake b/tools/cmake/Utils.cmake index e567fa503d4..3b42fe659a5 100644 --- a/tools/cmake/Utils.cmake +++ b/tools/cmake/Utils.cmake @@ -119,74 +119,6 @@ function(extract_sources sources_file) endif() endfunction() -# Sets the value of the BUCK2 variable by searching for a buck2 binary with the -# correct version. -# -# The resolve_buck.py script uses the following logic to find buck2: 1) If BUCK2 -# argument is set explicitly, use it. Warn if the version is incorrect. 2) Look -# for a binary named buck2 on the system path. Take it if it is the correct -# version. 3) Check for a previously downloaded buck2 binary (from step 4). 4) -# Download and cache correct version of buck2. -function(resolve_buck2) - if(EXECUTORCH_ROOT) - set(executorch_root ${EXECUTORCH_ROOT}) - else() - set(executorch_root ${CMAKE_CURRENT_SOURCE_DIR}) - endif() - - set(resolve_buck2_command - ${PYTHON_EXECUTABLE} ${executorch_root}/tools/cmake/resolve_buck.py - --cache_dir=${executorch_root}/buck2-bin - ) - - if(NOT ${BUCK2} STREQUAL "") - list(APPEND resolve_buck2_command --buck2=${BUCK2}) - endif() - - execute_process( - COMMAND ${resolve_buck2_command} - OUTPUT_VARIABLE resolve_buck2_output - ERROR_VARIABLE resolve_buck2_error - RESULT_VARIABLE resolve_buck2_exit_code - WORKING_DIRECTORY ${executorch_root} - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - - # $BUCK2 is a copy of the var from the parent scope. This block will set - # $buck2 to the value we want to return. - if(resolve_buck2_exit_code EQUAL 0) - set(buck2 ${resolve_buck2_output}) - message(STATUS "Resolved buck2 as ${resolve_buck2_output}.") - elseif(resolve_buck2_exit_code EQUAL 2) - # Wrong buck version used. Stop here to ensure that the user sees the error. - message(FATAL_ERROR "Failed to resolve buck2.\n${resolve_buck2_error}") - else() - # Unexpected failure of the script. Warn. - message(WARNING "Failed to resolve buck2.") - message(WARNING "${resolve_buck2_error}") - - if("${BUCK2}" STREQUAL "") - set(buck2 "buck2") - endif() - endif() - - # Update the var in the parent scope. Note that this does not modify our local - # $BUCK2 value. - set(BUCK2 - "${buck2}" - PARENT_SCOPE - ) - - # The buck2 daemon can get stuck. Killing it can help. - message(STATUS "Killing buck2 daemon") - execute_process( - # Note that we need to use the local buck2 variable. BUCK2 is only set in - # the parent scope, and can still be empty in this scope. - COMMAND "${buck2} killall" - WORKING_DIRECTORY ${executorch_root} COMMAND_ECHO STDOUT - ) -endfunction() - # Sets the value of the PYTHON_EXECUTABLE variable to 'python' if in an active # (non-base) conda environment, and 'python3' otherwise. This maintains # backwards compatibility for non-conda users and avoids conda users needing to From d412422d6c5934fb50679ffad9166ef7f37f1f98 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Wed, 13 Aug 2025 15:44:44 -0700 Subject: [PATCH 23/29] Update [ghstack-poisoned] --- .ci/scripts/utils.sh | 2 -- backends/cadence/build_cadence_fusionG3.sh | 4 ++-- backends/cadence/build_cadence_hifi4.sh | 4 ++-- examples/apple/mps/scripts/build_mps_executor_runner.sh | 2 +- examples/apple/mps/test_mps.sh | 8 +------- .../android/LlamaDemo/docs/delegates/mediatek_README.md | 1 - .../apple_ios/LLaMA/docs/delegates/xnnpack_README.md | 2 +- examples/qualcomm/test_qualcomm.sh | 8 +------- extension/llm/export/quantizer_lib.py | 2 +- test/build_optimized_size_test.sh | 2 +- test/build_size_test.sh | 2 +- 11 files changed, 11 insertions(+), 26 deletions(-) diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index 6902cc3dec1..f6f6ece786b 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -131,8 +131,6 @@ build_executorch_runner_cmake() { else CXXFLAGS="" fi - # This command uses buck2 to gather source files and buck2 could crash flakily - # on MacOS CXXFLAGS="$CXXFLAGS" retry cmake -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" -DCMAKE_BUILD_TYPE="${1:-Release}" .. popd || return diff --git a/backends/cadence/build_cadence_fusionG3.sh b/backends/cadence/build_cadence_fusionG3.sh index 1c84ae99364..93295bc9aa5 100644 --- a/backends/cadence/build_cadence_fusionG3.sh +++ b/backends/cadence/build_cadence_fusionG3.sh @@ -36,7 +36,7 @@ if $STEPWISE_BUILD; then -Bcmake-out . echo "Building any Cadence-specific binaries on top" - CXXFLAGS="-fno-exceptions -fno-rtti" cmake -DBUCK2="$BUCK" \ + CXXFLAGS="-fno-exceptions -fno-rtti" cmake \ -DCMAKE_TOOLCHAIN_FILE=/home/zonglinpeng/ws/zonglinpeng/executorch/backends/cadence/cadence.cmake \ -DCMAKE_INSTALL_PREFIX=cmake-out \ -DCMAKE_BUILD_TYPE=Release \ @@ -57,7 +57,7 @@ if $STEPWISE_BUILD; then else echo "Building Cadence toolchain with ExecuTorch packages" cmake_prefix_path="${PWD}/cmake-out/lib/cmake/ExecuTorch;${PWD}/cmake-out/third-party/gflags" - CXXFLAGS="-fno-exceptions -fno-rtti" cmake -DBUCK2="$BUCK" \ + CXXFLAGS="-fno-exceptions -fno-rtti" cmake \ -DCMAKE_PREFIX_PATH="${cmake_prefix_path}" \ -DHAVE_SYS_STAT_H=ON \ -DCMAKE_TOOLCHAIN_FILE=./backends/cadence/cadence.cmake \ diff --git a/backends/cadence/build_cadence_hifi4.sh b/backends/cadence/build_cadence_hifi4.sh index e0a48da4074..33078b7ff2f 100644 --- a/backends/cadence/build_cadence_hifi4.sh +++ b/backends/cadence/build_cadence_hifi4.sh @@ -35,7 +35,7 @@ if $STEPWISE_BUILD; then -Bcmake-out . echo "Building any Cadence-specific binaries on top" - CXXFLAGS="-fno-exceptions -fno-rtti" cmake -DBUCK2="$BUCK" \ + CXXFLAGS="-fno-exceptions -fno-rtti" cmake \ -DCMAKE_TOOLCHAIN_FILE=./backends/cadence/cadence.cmake \ -DCMAKE_INSTALL_PREFIX=cmake-out \ -DCMAKE_BUILD_TYPE=Release \ @@ -56,7 +56,7 @@ if $STEPWISE_BUILD; then else echo "Building Cadence toolchain with ExecuTorch packages" cmake_prefix_path="${PWD}/cmake-out/lib/cmake/ExecuTorch;${PWD}/cmake-out/third-party/gflags" - CXXFLAGS="-fno-exceptions -fno-rtti" cmake -DBUCK2="$BUCK" \ + CXXFLAGS="-fno-exceptions -fno-rtti" cmake \ -DCMAKE_PREFIX_PATH="${cmake_prefix_path}" \ -DCMAKE_TOOLCHAIN_FILE=./backends/cadence/cadence.cmake \ -DCMAKE_INSTALL_PREFIX=cmake-out \ diff --git a/examples/apple/mps/scripts/build_mps_executor_runner.sh b/examples/apple/mps/scripts/build_mps_executor_runner.sh index 625bc08a663..5d4e087d19e 100755 --- a/examples/apple/mps/scripts/build_mps_executor_runner.sh +++ b/examples/apple/mps/scripts/build_mps_executor_runner.sh @@ -38,7 +38,7 @@ done rm -rf "$OUTPUT" -cmake -DBUCK2="$BUCK" \ +cmake \ -DCMAKE_INSTALL_PREFIX=cmake-out \ -DCMAKE_BUILD_TYPE="$MODE" \ -DEXECUTORCH_BUILD_DEVTOOLS=ON \ diff --git a/examples/apple/mps/test_mps.sh b/examples/apple/mps/test_mps.sh index bca28628473..2d0507fcf56 100755 --- a/examples/apple/mps/test_mps.sh +++ b/examples/apple/mps/test_mps.sh @@ -15,7 +15,7 @@ cmake_install_executorch_devtools_lib() { echo "Installing libexecutorch.a, libportable_kernels.a, libetdump.a, libbundled_program.a" rm -rf cmake-out - retry cmake -DBUCK2="$BUCK" \ + retry cmake \ -DCMAKE_INSTALL_PREFIX=cmake-out \ -DCMAKE_BUILD_TYPE=Release \ -DEXECUTORCH_BUILD_DEVTOOLS=ON \ @@ -56,11 +56,5 @@ then PYTHON_EXECUTABLE=python3 fi -if [[ -z $BUCK ]]; -then - BUCK=buck2 -fi - - cmake_install_executorch_devtools_lib test_cmake_mps diff --git a/examples/demo-apps/android/LlamaDemo/docs/delegates/mediatek_README.md b/examples/demo-apps/android/LlamaDemo/docs/delegates/mediatek_README.md index 2ad87df0653..f72e1b0fbc7 100644 --- a/examples/demo-apps/android/LlamaDemo/docs/delegates/mediatek_README.md +++ b/examples/demo-apps/android/LlamaDemo/docs/delegates/mediatek_README.md @@ -54,7 +54,6 @@ zstd -cdq ".zst" > "/buck2" && chmod ### Set Environment Variables ``` -export BUCK2=path_to_buck/buck2 # Download BUCK2 and create BUCK2 executable export ANDROID_NDK=path_to_android_ndk export NEURON_BUFFER_ALLOCATOR_LIB=path_to_buffer_allocator/libneuron_buffer_allocator.so export NEURON_USDK_ADAPTER_LIB=path_to_usdk_adapter/libneuronusdk_adapter.mtk.so diff --git a/examples/demo-apps/apple_ios/LLaMA/docs/delegates/xnnpack_README.md b/examples/demo-apps/apple_ios/LLaMA/docs/delegates/xnnpack_README.md index 7a56d217b82..4ec10032c1f 100644 --- a/examples/demo-apps/apple_ios/LLaMA/docs/delegates/xnnpack_README.md +++ b/examples/demo-apps/apple_ios/LLaMA/docs/delegates/xnnpack_README.md @@ -127,7 +127,7 @@ Go to Project Navigator, click on LLaMA. `Project --> LLaMA --> Package Dependen Note: You should only use this step if the prebuilt package doesn't work for your usecase (For example, you require the latest PRs from main, where there are no pre-built package yet) -If you need to manually build the package, run the following command in your terminal +If you need to manually build the package, run the following command in your terminal: ``` # Install a compatible version of Buck2 BUCK2_RELEASE_DATE="2024-12-16" diff --git a/examples/qualcomm/test_qualcomm.sh b/examples/qualcomm/test_qualcomm.sh index 19d3d798418..51a563863f3 100644 --- a/examples/qualcomm/test_qualcomm.sh +++ b/examples/qualcomm/test_qualcomm.sh @@ -15,7 +15,7 @@ cmake_install_executorch_qnn_lib() { echo "Installing libexecutorch.a, libqnn_executorch_backend.a" rm -rf cmake-out - retry cmake -DBUCK2="$BUCK" \ + retry cmake \ -DCMAKE_INSTALL_PREFIX=cmake-out \ -DCMAKE_BUILD_TYPE=Release \ -DEXECUTORCH_BUILD_QNN=ON \ @@ -55,11 +55,5 @@ then PYTHON_EXECUTABLE=python3 fi -if [[ -z $BUCK ]]; -then - BUCK=buck2 -fi - - cmake_install_executorch_qnn_lib test_cmake_qualcomm diff --git a/extension/llm/export/quantizer_lib.py b/extension/llm/export/quantizer_lib.py index d87c722363f..2d87c86d113 100644 --- a/extension/llm/export/quantizer_lib.py +++ b/extension/llm/export/quantizer_lib.py @@ -108,7 +108,7 @@ def check_embedding_byte_registered(): "Need to specify shared library path to register quantized ops (and their out variants) into EXIR.\n" "Follow the following steps to build the needed lib via cmake.\n" "Then from root executorch dir do the following:\n" - "rm -rf cmake-out && mkdir cmake-out && (cd cmake-out && cmake -DBUCK2= -DEXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT=ON ..) && cmake --build . -j16\n" + "rm -rf cmake-out && mkdir cmake-out && (cd cmake-out && cmake -DEXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT=ON ..) && cmake --build . -j16\n" 'To find the location of the lib: find cmake-out -name "libquantized_ops_aot_lib*"\n' "Then specify the said library via -s Date: Wed, 13 Aug 2025 15:44:49 -0700 Subject: [PATCH 24/29] Update [ghstack-poisoned] --- pyproject.toml | 1 - requirements-dev.txt | 1 - tools/cmake/Utils.cmake | 52 ---- tools/cmake/cmake_deps.toml | 533 --------------------------------- tools/cmake/extract_sources.py | 255 ---------------- 5 files changed, 842 deletions(-) delete mode 100644 tools/cmake/cmake_deps.toml delete mode 100755 tools/cmake/extract_sources.py diff --git a/pyproject.toml b/pyproject.toml index 98cf935c191..61448a849cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,6 @@ requires = [ "pip>=23", # For building the pip package. "pyyaml", # Imported by the kernel codegen tools. "setuptools>=63", # For building the pip package contents. - "tomli", # Imported by extract_sources.py when using python < 3.11. "wheel", # For building the pip package archive. "zstd", # Imported by resolve_buck.py. "certifi", # Imported by resolve_buck.py. diff --git a/requirements-dev.txt b/requirements-dev.txt index e2a4f8af99e..9df5e7b93ed 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,6 @@ cmake>=3.29, <4.0.0 # For building binary targets in the wheel. pip>=23 # For building the pip package. pyyaml # Imported by the kernel codegen tools. setuptools>=63 # For building the pip package contents. -tomli # Imported by extract_sources.py when using python < 3.11. wheel # For building the pip package archive. zstd # Imported by resolve_buck.py. certifi # Imported by resolve_buck.py. diff --git a/tools/cmake/Utils.cmake b/tools/cmake/Utils.cmake index 3b42fe659a5..1e0671eb920 100644 --- a/tools/cmake/Utils.cmake +++ b/tools/cmake/Utils.cmake @@ -67,58 +67,6 @@ function(target_link_options_gc_sections target_name) endif() endfunction() -# Extract source files based on toml config. This is useful to keep buck2 and -# cmake aligned. Do not regenerate if file exists. -function(extract_sources sources_file) - if(EXISTS "${sources_file}") - message(STATUS "executorch: Using source file list ${sources_file}") - else() - # A file wasn't generated. Run a script to extract the source lists from the - # buck2 build system and write them to a file we can include. - # - # NOTE: This will only happen once during cmake setup, so it will not re-run - # if the buck2 targets change. - message(STATUS "executorch: Generating source file list ${sources_file}") - if(EXECUTORCH_ROOT) - set(executorch_root ${EXECUTORCH_ROOT}) - else() - set(executorch_root ${CMAKE_CURRENT_SOURCE_DIR}) - endif() - - if(ANDROID_ABI) - if("${ANDROID_ABI}" STREQUAL "arm64-v8a") - set(target_platforms_arg "--target-platforms=shim_et//:android-arm64") - elseif("${ANDROID_ABI}" STREQUAL "x86_64") - set(target_platforms_arg "--target-platforms=shim_et//:android-x86_64") - else() - message( - FATAL_ERROR - "Unsupported ANDROID_ABI setting ${ANDROID_ABI}. Please add it here!" - ) - endif() - endif() - execute_process( - COMMAND - ${PYTHON_EXECUTABLE} ${executorch_root}/tools/cmake/extract_sources.py - --config=${executorch_root}/tools/cmake/cmake_deps.toml - --out=${sources_file} --buck2=${BUCK2} ${target_platforms_arg} - OUTPUT_VARIABLE gen_srcs_output - ERROR_VARIABLE gen_srcs_error - RESULT_VARIABLE gen_srcs_exit_code - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) - - if(NOT gen_srcs_exit_code EQUAL 0) - message("Error while generating ${sources_file}. " - "Exit code: ${gen_srcs_exit_code}" - ) - message("Output:\n${gen_srcs_output}") - message("Error:\n${gen_srcs_error}") - message(FATAL_ERROR "executorch: source list generation failed") - endif() - endif() -endfunction() - # Sets the value of the PYTHON_EXECUTABLE variable to 'python' if in an active # (non-base) conda environment, and 'python3' otherwise. This maintains # backwards compatibility for non-conda users and avoids conda users needing to diff --git a/tools/cmake/cmake_deps.toml b/tools/cmake/cmake_deps.toml deleted file mode 100644 index 044f673f075..00000000000 --- a/tools/cmake/cmake_deps.toml +++ /dev/null @@ -1,533 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# Copyright 2024 Arm Limited and/or its affiliates. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# Inherited by all other targets. When a key already exists, the elements of the -# target's value are appended to lists here. -[target_base] -excludes = [ - "^third-party", -] - -# ---------------------------------- core start ---------------------------------- - -[targets.executorch] -buck_targets = [ - "//runtime/executor:program", -] -deps = [ - "executorch_core", -] -filters = [ - ".cpp$", -] - - -[targets.executorch_core] -buck_targets = [ - "//runtime/executor:program_no_prim_ops", -] -deps = [ - "program_schema", -] -filters = [ - ".cpp$", -] - - -[targets.portable_kernels] -buck_targets = [ - # //kernels/portable:operators would be more appropriate, but buck2 doesn't - # think it has any "inputs" since its srcs list is empty. - "//kernels/portable:generated_lib", -] -filters = [ - ".cpp$", -] -excludes = [ - # Exclude the codegen templates, which are picked up because the buck target - # is the generated_lib and not the unwrapped set of kernels. - "^codegen/templates", -] -deps = [ - "executorch", - "executorch_core", - "extension_threadpool", - "kernels_util_all_deps", -] - -[targets.kernels_util_all_deps] -buck_targets = [ - "//kernels/portable/cpu/util:all_deps", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", - "extension_threadpool", -] - -# HACK: prevent reduce_util from also showing up in custom_ops. The -# actual medium-term fix is to stop using Buck to drive our CMake -# builds. -[targets.reduce_util] -buck_targets = [ - "//kernels/portable/cpu/util:reduce_util", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch", - "executorch_core", -] - -[targets.optimized_kernels] -buck_targets = [ - "//kernels/optimized:generated_lib", -] -filters = [ - ".cpp$", -] -excludes = [ - # Exclude the codegen templates, which are picked up because the buck target - # is the generated_lib and not the unwrapped set of kernels. - "^codegen/templates", -] -deps = [ - "executorch", - "executorch_core", - "extension_threadpool", - "kernels_util_all_deps", - "optimized_cpublas", - "portable_kernels", -] - -[targets.quantized_kernels] -buck_targets = [ - "//kernels/quantized:generated_lib", -] -filters = [ - ".cpp$", -] -excludes = [ - # Exclude the codegen templates, which are picked up because the buck target - # is the generated_lib and not the unwrapped set of kernels. - "^codegen/templates", -] -deps = [ - "executorch", - "executorch_core", - "extension_threadpool", - "kernels_util_all_deps", - "portable_kernels", -] - -[targets.program_schema] -buck_targets = [ - "//schema:program", -] -filters = [ - ".fbs$", -] - -[targets.optimized_cpublas] -buck_targets = [ - "//kernels/optimized:libblas", -] -filters = [ - ".cpp$", -] -excludes = [ -] -deps = [ - "executorch_core", - "executorch", - "extension_threadpool", -] - -[targets.optimized_native_cpu_ops] -buck_targets = [ - "//configurations:optimized_native_cpu_ops", -] -filters = [ - ".cpp$", -] -excludes = [ -] -deps = [ - "executorch_core", - "executorch", - "extension_threadpool", - "kernels_util_all_deps", - "optimized_cpublas", - "portable_kernels", -] - -[targets.test_backend_compiler_lib] -buck_targets = [ - "//runtime/executor/test:test_backend_compiler_lib", -] -filters = [ - ".cpp$", -] -excludes = [ -] -deps = [ - "executorch", - "executorch_core", -] -# ---------------------------------- core end ---------------------------------- -# ---------------------------------- extension start ---------------------------------- -[targets.extension_data_loader] -buck_targets = [ - "//extension/data_loader:buffer_data_loader", - "//extension/data_loader:file_data_loader", - "//extension/data_loader:mmap_data_loader", - "//extension/data_loader:shared_ptr_data_loader", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_evalue_util] -buck_targets = [ - "//extension/evalue_util:print_evalue", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_flat_tensor_schema] -buck_targets = [ - "//extension/flat_tensor/serialize:generated_headers", -] -filters = [ - ".fbs$", -] - -[targets.extension_flat_tensor] -buck_targets = [ - "//extension/flat_tensor:flat_tensor_data_map", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_module] -buck_targets = [ - "//extension/module:module", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", - "extension_data_loader", - "extension_flat_tensor", -] - -[targets.extension_runner_util] -buck_targets = [ - "//extension/runner_util:inputs", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_tokenizers] -buck_targets = [ - "//extension/llm/tokenizers:sentencepiece", - "//extension/llm/tokenizers:tiktoken", - "//extension/llm/tokenizers:hf_tokenizer", - "//extension/llm/tokenizers:llama2c_tokenizer", -] -filters = [ - ".cpp$", -] - -[targets.extension_llm_runner] -buck_targets = [ - "//extension/llm/runner:runner_lib", - "//extension/llm/runner/io_manager:io_manager", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", - "extension_data_loader", - "extension_flat_tensor", - "extension_module", - "extension_data_loader", - "extension_flat_tensor", - "extension_runner_util", - "extension_tensor", - "extension_tokenizers", - "kernels_util_all_deps", -] - -[targets.extension_tensor] -buck_targets = [ - "//extension/tensor:tensor", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_threadpool] -buck_targets = [ - "//extension/threadpool:threadpool", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_training] -buck_targets = [ - "//extension/training/module:training_module", - "//extension/training/optimizer:sgd", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.train_xor] -buck_targets = [ - "//extension/training/examples/XOR:train_xor", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch", - "executorch_core", - "extension_threadpool", - "kernels_util_all_deps", - "portable_kernels", -] -# ---------------------------------- extension end ---------------------------------- -# ---------------------------------- binary start ---------------------------------- - -[targets.executor_runner] -buck_targets = [ - "//examples/portable/executor_runner:executor_runner", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch", - "executorch_core", - "extension_evalue_util", - "extension_runner_util", - "extension_threadpool", - "kernels_util_all_deps", - "portable_kernels", - "quantized_kernels", - "etdump_flatcc", -] - -[targets.size_test] -buck_targets = [ - "//test:size_test", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch_core", - "executorch", - "extension_data_loader", -] -# ---------------------------------- binary end ---------------------------------- -# ---------------------------------- MPS start ---------------------------------- -[targets.mps_executor_runner] -buck_targets = [ - "//examples/apple/mps/executor_runner:mps_executor_runner", -] -filters = [ - "(.mm|.cpp)$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch", - "executorch_core", - "extension_evalue_util", - "extension_runner_util", - "extension_threadpool", - "kernels_util_all_deps", - "portable_kernels", -] - -[targets.mps_backend] -buck_targets = [ - "//backends/apple/mps:mps", -] -filters = [ - "(.mm|.cpp)$", -] -deps = [ - "executorch", - "executorch_core", -] - -[targets.mps_schema] -buck_targets = [ - "//backends/apple/mps:mps_schema", -] -filters = [ - ".fbs$", -] - -# ---------------------------------- MPS end ---------------------------------- -# ---------------------------------- XNNPACK start ---------------------------------- - -[targets.xnn_executor_runner] -buck_targets = [ - "//examples/xnnpack:xnn_executor_runner", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch", - "executorch_core", - "extension_evalue_util", - "extension_runner_util", - "extension_threadpool", - "kernels_util_all_deps", - "xnnpack_backend", - "portable_kernels", - "etdump_flatcc", -] - -[targets.xnnpack_backend] -buck_targets = [ - "//backends/xnnpack:xnnpack_backend", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch", - "executorch_core", - "extension_threadpool", -] - -[targets.xnnpack_schema] -buck_targets = [ - "//backends/xnnpack/serialization:xnnpack_flatbuffer_header", -] -filters = [ - ".fbs$", -] -# ---------------------------------- XNNPACK end ---------------------------------- -# ---------------------------------- Vulkan start --------------------------------- -[targets.vulkan_schema] -buck_targets = [ - "//backends/vulkan/serialization:vk_delegate_schema", -] -filters = [ - ".fbs$", -] -# ---------------------------------- Vulkan end ----------------------------------- -# ---------------------------------- LLama start ---------------------------------- -[targets.custom_ops] -buck_targets = [ - "//extension/llm/custom_ops:custom_ops", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch", - "executorch_core", - "optimized_cpublas", - "optimized_kernels", - "extension_threadpool", - "reduce_util", - "xnnpack_backend", -] - -[targets.llama_runner] -buck_targets = [ - "//examples/models/llama/runner:runner", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "custom_ops", - "executorch", - "executorch_core", - "extension_data_loader", - "extension_flat_tensor", - "extension_llm_runner", - "extension_module", - "extension_tensor", - "extension_threadpool", - "extension_tokenizers", - "kernels_util_all_deps", - "optimized_cpublas", - "portable_kernels", - "quantized_kernels", - "xnnpack_backend", - "optimized_native_cpu_ops", -] -# ---------------------------------- LLama end ---------------------------------- -# ---------------------------------- devtools start ---------------------------------- -[targets.etdump_flatcc] -buck_targets = [ - "//devtools/etdump:etdump_flatcc", -] -filters = [ - ".cpp$", -] -# ---------------------------------- devtools end ---------------------------------- diff --git a/tools/cmake/extract_sources.py b/tools/cmake/extract_sources.py deleted file mode 100755 index 5af0904fdfd..00000000000 --- a/tools/cmake/extract_sources.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import copy -import logging -import os -import re - -from enum import Enum -from typing import Any, List, Optional, Sequence - -from buck_util import Buck2Runner - -try: - import tomllib # Standard in 3.11 and later -except ModuleNotFoundError: - import tomli as tomllib # type: ignore[no-redef] - -"""Extracts source lists from the buck2 build system and writes them to a file. - -The config file is in TOML format and should contains one or more -`[targets.]` entries, along with an optional `[target_base]` entry. - -All of these may have the following lists of strings: -- buck_targets: The list of buck targets that map to ``. -- deps: A list of other `` entries that this target depends on. - Used to prune sources that are provided by those other targets. -- filters: A list of regular expressions. This tool will only emit source files - whose relative paths match all entries. -- excludes: A list of regular expressions. This tool will not emit source files - whose relative paths match any entry. - -The special `[target_base]` entry provides default lists that are inherited by -the `[target.]` entries. When the `[target.]` entry defines -a key that is already present in `[target_base]`, the target-specific entries are -appended to the base list. - -Example config: - - [target_base] - excludes = [ - "^third-party", - ] - - [targets.schema] - buck_targets = [ - "//schema:schema", - ] - filters = [ - ".fbs$", - ] - - [targets.executorch] - buck_targets = [ - "//runtime/executor:program", - ] - deps = [ - "schema", - ] - filters = [ - ".cpp$", - ] -""" - -# Set up logging -logging.basicConfig( - level=logging.INFO, format="%(asctime)s [ExecuTorch] %(levelname)s: %(message)s" -) -logger = logging.getLogger() - - -class Target: - """Parsed [targets.*] entry from the TOML file. - - Can query buck for its list of source files. - """ - - class _InitState(Enum): - UNINITIALIZED = 0 - INITIALIZING = 1 - READY = 2 - - def __init__( - self, - name: str, - target_dict: dict[str, Sequence[str]], - base_dict: Optional[dict] = None, - ) -> None: - self._state: Target._InitState = Target._InitState.UNINITIALIZED - self._sources: frozenset[str] = frozenset() - - self.name = name - # Extend the base lists with the target-specific entries. - self._config = copy.deepcopy(base_dict or {}) - for k, v in target_dict.items(): - if k in self._config: - self._config[k].extend(v) - else: - self._config[k] = v - - def get_sources( - self, graph: "Graph", runner: Buck2Runner, buck_args: Optional[List[str]] - ) -> frozenset[str]: - if buck_args is None: - buck_args = [] - - if self._state == Target._InitState.READY: - return self._sources - # Detect cycles. - assert self._state != Target._InitState.INITIALIZING - - # Assemble the query. - query = "inputs({})".format( - "+".join( - [ - "deps('{}')".format(target) - for target in self._config.get("buck_targets", []) - ] - ) - ) - - # Get the complete list of source files that this target depends on. - # If user doesn't setup their git submodules correctly, this will fail. - # If we hit here, setup.py:check_submodule() should have already run - # but it could be that the submodules are not synced or there's local changes. - try: - sources: set[str] = set(runner.run(["cquery", query] + buck_args)) - except RuntimeError as e: - logger.error( - f"\033[31;1mFailed to query buck for sources. Failed command:\n\n" - f" buck2 cquery {query} {' '.join(buck_args)}\n\n" - "This is likely due " - "to missing git submodules or outdated CMake cache. " - "Please run the following before retry:\033[0m\n\n" - " \033[32;1m./install_executorch.sh --clean\033[0m\n" - " \033[32;1mgit submodule sync\033[0m\n" - " \033[32;1mgit submodule update --init\033[0m\n" - ) - raise e - - # Keep entries that match all of the filters. - filters = [re.compile(p) for p in self._config.get("filters", [])] - sources = {s for s in sources if all(p.search(s) for p in filters)} - - # Remove entries that match any of the excludes. - excludes = [re.compile(p) for p in self._config.get("excludes", [])] - sources = {s for s in sources if not any(p.search(s) for p in excludes)} - - # The buck query will give us the complete list of sources that this - # target depends on, but that list includes sources that are owned by - # its deps. Remove entries that are already covered by the transitive - # set of dependencies. - for dep in self._config.get("deps", []): - sources.difference_update( - graph.by_name[dep].get_sources(graph, runner, buck_args) - ) - - self._sources = frozenset(sources) - self._state = Target._InitState.READY - return self._sources - - -class Graph: - """Graph of targets.""" - - def __init__(self, config_dict: dict[str, Any]) -> None: - base = config_dict.get("target_base", {}) - targets = config_dict.get("targets", {}) - - self.by_name = {} - for k, v in targets.items(): - self.by_name[k] = Target(k, v, base) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Extracts deps from the buck2 build system", - ) - parser.add_argument( - "--buck2", - default="buck2", - help="'buck2' command to use", - ) - parser.add_argument( - "--config", - metavar="config.toml", - required=True, - help="Path to the input TOML configuration file", - ) - parser.add_argument( - "--format", - default="cmake", - choices=["cmake"], - help="Format to generate.", - ) - parser.add_argument( - "--out", - metavar="file", - help="Path to the file to generate.", - ) - parser.add_argument( - "--target-platforms", help="--target-platforms to pass to buck cquery, if any." - ) - return parser.parse_args() - - -def generate_cmake(target_to_srcs: dict[str, list[str]]) -> bytes: - lines: list[str] = [] - lines.append("# @" + f"generated by {os.path.basename(__file__)}") - for target, srcs in target_to_srcs.items(): - lines.append("") - lines.append(f"set(_{target}__srcs") - for src in srcs: - lines.append(f" {src}") - lines.append(")") - return "\n".join(lines).encode("utf-8") - - -def main(): - args = parse_args() - - # Load and parse the TOML configuration - with open(args.config, mode="rb") as fp: - config_dict = tomllib.load(fp) - graph = Graph(config_dict) - - # Run the queries and get the lists of source files. - target_to_srcs: dict[str, list[str]] = {} - runner: Buck2Runner = Buck2Runner(args.buck2) - buck_args = [] - if args.target_platforms: - buck_args = ["--target-platforms"] - buck_args.append(args.target_platforms) - for name, target in graph.by_name.items(): - target_to_srcs[name] = sorted(target.get_sources(graph, runner, buck_args)) - - # Generate the requested format. - output: bytes - if args.format == "cmake": - output = generate_cmake(target_to_srcs) - else: - raise ValueError("Unknown format: {}".format(args.format)) - - # Write the output. - with open(args.out, "wb") as fp: - fp.write(output) - - -if __name__ == "__main__": - main() From 97ac2175330ace20d4a3c46f2fccd95a93fcdf13 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Thu, 14 Aug 2025 13:51:03 -0700 Subject: [PATCH 25/29] rebase [ghstack-poisoned] --- shim_et/xplat/executorch/build/build_variables.bzl | 5 ----- tools/cmake/Codegen.cmake | 2 -- 2 files changed, 7 deletions(-) diff --git a/shim_et/xplat/executorch/build/build_variables.bzl b/shim_et/xplat/executorch/build/build_variables.bzl index aa8ad0d4003..95f9cec0d42 100644 --- a/shim_et/xplat/executorch/build/build_variables.bzl +++ b/shim_et/xplat/executorch/build/build_variables.bzl @@ -289,11 +289,6 @@ QUANTIZED_KERNELS_SRCS = [ "kernels/quantized/cpu/op_quantize.cpp", ] -PROGRAM_SCHEMA_SRCS = [ - "schema/program.fbs", - "schema/scalar_type.fbs", -] - OPTIMIZED_CPUBLAS_SRCS = [ "kernels/optimized/blas/BlasKernel.cpp", "kernels/optimized/blas/CPUBlas.cpp", diff --git a/tools/cmake/Codegen.cmake b/tools/cmake/Codegen.cmake index aa9c2133851..3511592daa7 100644 --- a/tools/cmake/Codegen.cmake +++ b/tools/cmake/Codegen.cmake @@ -386,7 +386,6 @@ function(executorch_load_build_variables) KERNELS_UTIL_ALL_DEPS_SRCS OPTIMIZED_KERNELS_SRCS QUANTIZED_KERNELS_SRCS - PROGRAM_SCHEMA_SRCS OPTIMIZED_CPUBLAS_SRCS OPTIMIZED_NATIVE_CPU_OPS_SRCS TEST_BACKEND_COMPILER_LIB_SRCS @@ -419,7 +418,6 @@ function(executorch_load_build_variables) _kernels_util_all_deps__srcs _optimized_kernels__srcs _quantized_kernels__srcs - _program_schema__srcs _optimized_cpublas__srcs _optimized_native_cpu_ops__srcs _test_backend_compiler_lib__srcs From 0197f62d77b5021ec15d9e798f1af4293fdcf101 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Thu, 14 Aug 2025 13:51:07 -0700 Subject: [PATCH 26/29] rebase [ghstack-poisoned] --- .../xplat/executorch/build/build_variables.bzl | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/shim_et/xplat/executorch/build/build_variables.bzl b/shim_et/xplat/executorch/build/build_variables.bzl index 95f9cec0d42..8f87e3aceed 100644 --- a/shim_et/xplat/executorch/build/build_variables.bzl +++ b/shim_et/xplat/executorch/build/build_variables.bzl @@ -367,27 +367,14 @@ THREADPOOL_SRCS = [ EXTENSION_THREADPOOL_SRCS = ["extension/threadpool/" + x for x in THREADPOOL_SRCS] EXTENSION_TRAINING_SRCS = [ - "extension/data_loader/file_data_loader.cpp", - "extension/data_loader/mmap_data_loader.cpp", - "extension/flat_tensor/flat_tensor_data_map.cpp", - "extension/flat_tensor/serialize/flat_tensor_header.cpp", - "extension/module/module.cpp", "extension/training/module/training_module.cpp", "extension/training/optimizer/sgd.cpp", ] TRAIN_XOR_SRCS = [ - "extension/data_loader/file_data_loader.cpp", - "extension/data_loader/mmap_data_loader.cpp", - "extension/flat_tensor/flat_tensor_data_map.cpp", - "extension/flat_tensor/serialize/flat_tensor_header.cpp", + # REVIEW: removing this breaks the build; where is it supposed to come from? "extension/flat_tensor/serialize/serialize.cpp", - "extension/module/module.cpp", - "extension/tensor/tensor_ptr.cpp", - "extension/tensor/tensor_ptr_maker.cpp", "extension/training/examples/XOR/train.cpp", - "extension/training/module/training_module.cpp", - "extension/training/optimizer/sgd.cpp", ] EXECUTOR_RUNNER_SRCS = [ From 7b97a9ff2b25ffcf04527333e0d78e970a766807 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Mon, 18 Aug 2025 12:33:23 -0700 Subject: [PATCH 27/29] Update [ghstack-poisoned] --- .ci/scripts/unittest-buck2.sh | 7 ++++--- extension/flat_tensor/targets.bzl | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.ci/scripts/unittest-buck2.sh b/.ci/scripts/unittest-buck2.sh index f56db8924be..658fafdfcca 100755 --- a/.ci/scripts/unittest-buck2.sh +++ b/.ci/scripts/unittest-buck2.sh @@ -11,9 +11,10 @@ set -eux # TODO: can't query //kernels/prim_ops because of non-buckified stuff in OSS. buck2 query "//backends/apple/... + //backends/example/... + \ //backends/mediatek/... + //backends/transforms/... + \ -//backends/xnnpack/... + //configurations/... + //kernels/aten/... + \ -//kernels/optimized/... + //kernels/portable/... + //kernels/quantized/... + \ -//kernels/test/... + //runtime/... + //schema/... + //test/... + //util/..." +//backends/xnnpack/... + //configurations/... + //extension/flat_tensor: + \ +//kernels/aten/... + //kernels/optimized/... + //kernels/portable/... + \ +//kernels/quantized/... + //kernels/test/... + //runtime/... + //schema/... \ ++ //test/... + //util/..." # TODO: optimized ops are unbuildable because they now use ATen; put # them back after we can use PyTorch in OSS buck. diff --git a/extension/flat_tensor/targets.bzl b/extension/flat_tensor/targets.bzl index 3bc36dad9d4..f91e28a2268 100644 --- a/extension/flat_tensor/targets.bzl +++ b/extension/flat_tensor/targets.bzl @@ -1,7 +1,7 @@ -load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_aten_mode_options", "runtime") def define_common_targets(): - for aten_mode in [True, False]: + for aten_mode in get_aten_mode_options(): aten_suffix = "_aten" if aten_mode else "" runtime.cxx_library( name = "flat_tensor_data_map" + aten_suffix, From 0114056c5119c7a7b6f850b7256f1923f896d88c Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Mon, 18 Aug 2025 12:33:28 -0700 Subject: [PATCH 28/29] Update [ghstack-poisoned] --- .ci/scripts/unittest-macos-cmake.sh | 1 + CMakeLists.txt | 9 ++++ ..._installed_private_headers_in_cmake_out.sh | 44 +++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 test/check_for_installed_private_headers_in_cmake_out.sh diff --git a/.ci/scripts/unittest-macos-cmake.sh b/.ci/scripts/unittest-macos-cmake.sh index cdb40c40244..1a6cd2a15f2 100755 --- a/.ci/scripts/unittest-macos-cmake.sh +++ b/.ci/scripts/unittest-macos-cmake.sh @@ -11,3 +11,4 @@ ${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml # Run gtest LLVM_PROFDATA="xcrun llvm-profdata" LLVM_COV="xcrun llvm-cov" \ ${CONDA_RUN} test/run_oss_cpp_tests.sh +${CONDA_RUN} test/check_for_installed_private_headers_in_cmake_out.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f59c259332..9661d73c3d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -485,24 +485,29 @@ install( DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/runtime/core FILES_MATCHING PATTERN "*.h" + PATTERN "testing_util" EXCLUDE ) install( DIRECTORY runtime/executor/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/runtime/executor FILES_MATCHING PATTERN "*.h" + PATTERN "test" EXCLUDE + PATTERN "platform_memory_allocator.h" EXCLUDE ) install( DIRECTORY runtime/kernel/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/runtime/kernel FILES_MATCHING PATTERN "*.h" + PATTERN "test" EXCLUDE ) install( DIRECTORY runtime/platform/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/runtime/platform FILES_MATCHING PATTERN "*.h" + PATTERN "test" EXCLUDE ) install( DIRECTORY extension/kernel_util/ @@ -587,11 +592,15 @@ endif() if(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/data_loader) + if(NOT WIN32) + set(data_loader_exclude_pattern "*mman_windows.h") + endif() install( DIRECTORY extension/data_loader/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/extension/data_loader FILES_MATCHING PATTERN "*.h" + PATTERN ${data_loader_exclude_pattern} EXCLUDE ) list(APPEND _executorch_extensions extension_data_loader) endif() diff --git a/test/check_for_installed_private_headers_in_cmake_out.sh b/test/check_for_installed_private_headers_in_cmake_out.sh new file mode 100644 index 00000000000..74fc6d719d7 --- /dev/null +++ b/test/check_for_installed_private_headers_in_cmake_out.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# This script verifies that all headers that are installed under +# cmake-out/include/executorch are exported_headers of some Buck +# target. (It does *not* verify the reverse, namely that all +# exported_headers of every Buck target that should have been built +# when that directory was installed are actually installed.) +# +# Ideally, "some Buck target" would include any target in the whole +# repo, but we cannot yet buck query the whole repo. (See +# .ci/scripts/unittest-buck2.sh.) Instead, we query a manually-curated +# list of targets. + +set -euxo pipefail + +BUCK_HEADERS_TEMPFILE=$(mktemp /tmp/check_private_headers_buck.txt.XXXXXX) +ACTUAL_HEADERS_TEMPFILE=$(mktemp /tmp/check_private_headers_installed.txt.XXXXXX) +SOURCE_ROOT_DIR=$(git rev-parse --show-toplevel) +BUCK2=$(python3 "${SOURCE_ROOT_DIR}/tools/cmake/resolve_buck.py" --cache_dir="${SOURCE_ROOT_DIR}/buck2-bin") +if [[ "$BUCK2" == "buck2" ]]; then + BUCK2=$(command -v buck2) +fi + +"${SOURCE_ROOT_DIR}/scripts/print_exported_headers.py" \ + --buck2=$(realpath "$BUCK2") --targets \ + //extension/data_loader: //extension/evalue_util: \ + //extension/flat_tensor: //extension/kernel_util: //extension/module: \ + //extension/runner_util: //extension/tensor: //extension/threadpool: \ + | sort > "${BUCK_HEADERS_TEMPFILE}" +find "${SOURCE_ROOT_DIR}/cmake-out/include/executorch" -name '*.h' | \ + sed -e "s|${SOURCE_ROOT_DIR}/cmake-out/include/executorch/||" | \ + # Don't complain about generated Functions.h \ + grep -E -v 'Functions.h$' | sort > "${ACTUAL_HEADERS_TEMPFILE}" +ACTUAL_HEADERS_NOT_EXPORTED_IN_BUCK=$(comm -13 "${BUCK_HEADERS_TEMPFILE}" "${ACTUAL_HEADERS_TEMPFILE}") +if [[ -n "${ACTUAL_HEADERS_NOT_EXPORTED_IN_BUCK}" ]]; then + >&2 echo "The following non-exported headers were installed: +${ACTUAL_HEADERS_NOT_EXPORTED_IN_BUCK}" + exit 1 +fi From 7a6747fab05148be18f037b9cc6c0ce069ccf390 Mon Sep 17 00:00:00 2001 From: Scott Wolchok Date: Tue, 19 Aug 2025 15:37:57 -0700 Subject: [PATCH 29/29] Update [ghstack-poisoned] --- .ci/scripts/unittest-buck2.sh | 6 +++--- extension/llm/runner/io_manager/targets.bzl | 4 ++-- extension/llm/runner/targets.bzl | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/scripts/unittest-buck2.sh b/.ci/scripts/unittest-buck2.sh index 658fafdfcca..f748be62ac1 100755 --- a/.ci/scripts/unittest-buck2.sh +++ b/.ci/scripts/unittest-buck2.sh @@ -12,9 +12,9 @@ set -eux buck2 query "//backends/apple/... + //backends/example/... + \ //backends/mediatek/... + //backends/transforms/... + \ //backends/xnnpack/... + //configurations/... + //extension/flat_tensor: + \ -//kernels/aten/... + //kernels/optimized/... + //kernels/portable/... + \ -//kernels/quantized/... + //kernels/test/... + //runtime/... + //schema/... \ -+ //test/... + //util/..." +//extension/llm/runner: + //kernels/aten/... + //kernels/optimized/... + \ +//kernels/portable/... + //kernels/quantized/... + //kernels/test/... + \ +//runtime/... + //schema/... + //test/... + //util/..." # TODO: optimized ops are unbuildable because they now use ATen; put # them back after we can use PyTorch in OSS buck. diff --git a/extension/llm/runner/io_manager/targets.bzl b/extension/llm/runner/io_manager/targets.bzl index e538572c51b..ef93d541098 100644 --- a/extension/llm/runner/io_manager/targets.bzl +++ b/extension/llm/runner/io_manager/targets.bzl @@ -1,8 +1,8 @@ -load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_aten_mode_options", "runtime") def define_common_targets(): - for aten in (True, False): + for aten in get_aten_mode_options(): aten_suffix = "_aten" if aten else "" # Interface for IOManager. No concrete impl from this dep. diff --git a/extension/llm/runner/targets.bzl b/extension/llm/runner/targets.bzl index a6c17f3037c..05f05ac6fad 100644 --- a/extension/llm/runner/targets.bzl +++ b/extension/llm/runner/targets.bzl @@ -1,4 +1,4 @@ -load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_aten_mode_options", "runtime") def define_common_targets(): runtime.cxx_library( @@ -32,7 +32,7 @@ def define_common_targets(): ], ) - for aten in (True, False): + for aten in get_aten_mode_options(): aten_suffix = "_aten" if aten else "" runtime.cxx_library(