diff --git a/pyproject.toml b/pyproject.toml index 98cf935c191..61448a849cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,6 @@ requires = [ "pip>=23", # For building the pip package. "pyyaml", # Imported by the kernel codegen tools. "setuptools>=63", # For building the pip package contents. - "tomli", # Imported by extract_sources.py when using python < 3.11. "wheel", # For building the pip package archive. "zstd", # Imported by resolve_buck.py. "certifi", # Imported by resolve_buck.py. diff --git a/requirements-dev.txt b/requirements-dev.txt index e2a4f8af99e..9df5e7b93ed 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,6 @@ cmake>=3.29, <4.0.0 # For building binary targets in the wheel. pip>=23 # For building the pip package. pyyaml # Imported by the kernel codegen tools. setuptools>=63 # For building the pip package contents. -tomli # Imported by extract_sources.py when using python < 3.11. wheel # For building the pip package archive. zstd # Imported by resolve_buck.py. certifi # Imported by resolve_buck.py. diff --git a/shim_et/xplat/executorch/build/build_variables.bzl b/shim_et/xplat/executorch/build/build_variables.bzl index 81738becdc8..92a6f674279 100644 --- a/shim_et/xplat/executorch/build/build_variables.bzl +++ b/shim_et/xplat/executorch/build/build_variables.bzl @@ -290,11 +290,6 @@ QUANTIZED_KERNELS_SRCS = [ "kernels/quantized/cpu/op_quantize.cpp", ] -PROGRAM_SCHEMA_SRCS = [ - "schema/program.fbs", - "schema/scalar_type.fbs", -] - OPTIMIZED_CPUBLAS_SRCS = [ "kernels/optimized/blas/BlasKernel.cpp", "kernels/optimized/blas/CPUBlas.cpp", diff --git a/tools/cmake/Codegen.cmake b/tools/cmake/Codegen.cmake index aa9c2133851..3511592daa7 100644 --- a/tools/cmake/Codegen.cmake +++ b/tools/cmake/Codegen.cmake @@ -386,7 +386,6 @@ function(executorch_load_build_variables) KERNELS_UTIL_ALL_DEPS_SRCS OPTIMIZED_KERNELS_SRCS QUANTIZED_KERNELS_SRCS - PROGRAM_SCHEMA_SRCS OPTIMIZED_CPUBLAS_SRCS OPTIMIZED_NATIVE_CPU_OPS_SRCS TEST_BACKEND_COMPILER_LIB_SRCS @@ -419,7 +418,6 @@ function(executorch_load_build_variables) _kernels_util_all_deps__srcs _optimized_kernels__srcs _quantized_kernels__srcs - _program_schema__srcs _optimized_cpublas__srcs _optimized_native_cpu_ops__srcs _test_backend_compiler_lib__srcs diff --git a/tools/cmake/Utils.cmake b/tools/cmake/Utils.cmake index 3b42fe659a5..1e0671eb920 100644 --- a/tools/cmake/Utils.cmake +++ b/tools/cmake/Utils.cmake @@ -67,58 +67,6 @@ function(target_link_options_gc_sections target_name) endif() endfunction() -# Extract source files based on toml config. This is useful to keep buck2 and -# cmake aligned. Do not regenerate if file exists. -function(extract_sources sources_file) - if(EXISTS "${sources_file}") - message(STATUS "executorch: Using source file list ${sources_file}") - else() - # A file wasn't generated. Run a script to extract the source lists from the - # buck2 build system and write them to a file we can include. - # - # NOTE: This will only happen once during cmake setup, so it will not re-run - # if the buck2 targets change. - message(STATUS "executorch: Generating source file list ${sources_file}") - if(EXECUTORCH_ROOT) - set(executorch_root ${EXECUTORCH_ROOT}) - else() - set(executorch_root ${CMAKE_CURRENT_SOURCE_DIR}) - endif() - - if(ANDROID_ABI) - if("${ANDROID_ABI}" STREQUAL "arm64-v8a") - set(target_platforms_arg "--target-platforms=shim_et//:android-arm64") - elseif("${ANDROID_ABI}" STREQUAL "x86_64") - set(target_platforms_arg "--target-platforms=shim_et//:android-x86_64") - else() - message( - FATAL_ERROR - "Unsupported ANDROID_ABI setting ${ANDROID_ABI}. Please add it here!" - ) - endif() - endif() - execute_process( - COMMAND - ${PYTHON_EXECUTABLE} ${executorch_root}/tools/cmake/extract_sources.py - --config=${executorch_root}/tools/cmake/cmake_deps.toml - --out=${sources_file} --buck2=${BUCK2} ${target_platforms_arg} - OUTPUT_VARIABLE gen_srcs_output - ERROR_VARIABLE gen_srcs_error - RESULT_VARIABLE gen_srcs_exit_code - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) - - if(NOT gen_srcs_exit_code EQUAL 0) - message("Error while generating ${sources_file}. " - "Exit code: ${gen_srcs_exit_code}" - ) - message("Output:\n${gen_srcs_output}") - message("Error:\n${gen_srcs_error}") - message(FATAL_ERROR "executorch: source list generation failed") - endif() - endif() -endfunction() - # Sets the value of the PYTHON_EXECUTABLE variable to 'python' if in an active # (non-base) conda environment, and 'python3' otherwise. This maintains # backwards compatibility for non-conda users and avoids conda users needing to diff --git a/tools/cmake/cmake_deps.toml b/tools/cmake/cmake_deps.toml deleted file mode 100644 index cf9951e71f1..00000000000 --- a/tools/cmake/cmake_deps.toml +++ /dev/null @@ -1,558 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# Copyright 2024 Arm Limited and/or its affiliates. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# Inherited by all other targets. When a key already exists, the elements of the -# target's value are appended to lists here. -[target_base] -excludes = [ - "^third-party", -] - -# ---------------------------------- core start ---------------------------------- - -[targets.executorch] -buck_targets = [ - "//runtime/executor:program", -] -deps = [ - "executorch_core", -] -filters = [ - ".cpp$", -] - - -[targets.executorch_core] -buck_targets = [ - "//runtime/executor:program_no_prim_ops", -] -deps = [ - "program_schema", -] -filters = [ - ".cpp$", -] - - -[targets.portable_kernels] -buck_targets = [ - # //kernels/portable:operators would be more appropriate, but buck2 doesn't - # think it has any "inputs" since its srcs list is empty. - "//kernels/portable:generated_lib", -] -filters = [ - ".cpp$", -] -excludes = [ - # Exclude the codegen templates, which are picked up because the buck target - # is the generated_lib and not the unwrapped set of kernels. - "^codegen/templates", -] -deps = [ - "executorch", - "executorch_core", - "extension_threadpool", - "kernels_util_all_deps", -] - -[targets.kernels_util_all_deps] -buck_targets = [ - "//kernels/portable/cpu/util:all_deps", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", - "extension_threadpool", -] - -# HACK: prevent reduce_util from also showing up in custom_ops. The -# actual medium-term fix is to stop using Buck to drive our CMake -# builds. -[targets.reduce_util] -buck_targets = [ - "//kernels/portable/cpu/util:reduce_util", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch", - "executorch_core", -] - -[targets.optimized_kernels] -buck_targets = [ - "//kernels/optimized:generated_lib", -] -filters = [ - ".cpp$", -] -excludes = [ - # Exclude the codegen templates, which are picked up because the buck target - # is the generated_lib and not the unwrapped set of kernels. - "^codegen/templates", -] -deps = [ - "executorch", - "executorch_core", - "extension_threadpool", - "kernels_util_all_deps", - "optimized_cpublas", - "portable_kernels", -] - -[targets.quantized_kernels] -buck_targets = [ - "//kernels/quantized:generated_lib", -] -filters = [ - ".cpp$", -] -excludes = [ - # Exclude the codegen templates, which are picked up because the buck target - # is the generated_lib and not the unwrapped set of kernels. - "^codegen/templates", -] -deps = [ - "executorch", - "executorch_core", - "extension_threadpool", - "kernels_util_all_deps", - "portable_kernels", -] - -[targets.program_schema] -buck_targets = [ - "//schema:program", -] -filters = [ - ".fbs$", -] - -[targets.optimized_cpublas] -buck_targets = [ - "//kernels/optimized:libblas", -] -filters = [ - ".cpp$", -] -excludes = [ -] -deps = [ - "executorch_core", - "executorch", - "extension_threadpool", -] - -[targets.optimized_native_cpu_ops] -buck_targets = [ - "//configurations:optimized_native_cpu_ops", -] -filters = [ - ".cpp$", -] -excludes = [ -] -deps = [ - "executorch_core", - "executorch", - "extension_threadpool", - "kernels_util_all_deps", - "optimized_cpublas", - "portable_kernels", -] - -[targets.test_backend_compiler_lib] -buck_targets = [ - "//runtime/executor/test:test_backend_compiler_lib", -] -filters = [ - ".cpp$", -] -excludes = [ -] -deps = [ - "executorch", - "executorch_core", -] -# ---------------------------------- core end ---------------------------------- -# ---------------------------------- extension start ---------------------------------- -[targets.extension_data_loader] -buck_targets = [ - "//extension/data_loader:buffer_data_loader", - "//extension/data_loader:file_data_loader", - "//extension/data_loader:mmap_data_loader", - "//extension/data_loader:shared_ptr_data_loader", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_evalue_util] -buck_targets = [ - "//extension/evalue_util:print_evalue", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_flat_tensor_schema] -buck_targets = [ - "//extension/flat_tensor/serialize:generated_headers", -] -filters = [ - ".fbs$", -] - -[targets.extension_flat_tensor] -buck_targets = [ - "//extension/flat_tensor:flat_tensor_data_map", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_module] -buck_targets = [ - "//extension/module:module", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", - "extension_data_loader", - "extension_flat_tensor", -] - -[targets.bundled_module] -buck_targets = [ - "//extension/module:bundled_module", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", - "extension_data_loader", - "extension_module", - "bundled_program", -] - -[targets.extension_runner_util] -buck_targets = [ - "//extension/runner_util:inputs", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_tokenizers] -buck_targets = [ - "//extension/llm/tokenizers:sentencepiece", - "//extension/llm/tokenizers:tiktoken", - "//extension/llm/tokenizers:hf_tokenizer", - "//extension/llm/tokenizers:llama2c_tokenizer", -] -filters = [ - ".cpp$", -] - -[targets.extension_llm_runner] -buck_targets = [ - "//extension/llm/runner:runner_lib", - "//extension/llm/runner/io_manager:io_manager", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", - "extension_data_loader", - "extension_flat_tensor", - "extension_module", - "extension_data_loader", - "extension_flat_tensor", - "extension_runner_util", - "extension_tensor", - "extension_tokenizers", - "kernels_util_all_deps", -] - -[targets.extension_tensor] -buck_targets = [ - "//extension/tensor:tensor", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_threadpool] -buck_targets = [ - "//extension/threadpool:threadpool", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.extension_training] -buck_targets = [ - "//extension/training/module:training_module", - "//extension/training/optimizer:sgd", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.train_xor] -buck_targets = [ - "//extension/training/examples/XOR:train_xor", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch", - "executorch_core", - "extension_threadpool", - "kernels_util_all_deps", - "portable_kernels", -] -# ---------------------------------- extension end ---------------------------------- -# ---------------------------------- binary start ---------------------------------- - -[targets.executor_runner] -buck_targets = [ - "//examples/portable/executor_runner:executor_runner", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch", - "executorch_core", - "extension_evalue_util", - "extension_runner_util", - "extension_threadpool", - "kernels_util_all_deps", - "portable_kernels", - "quantized_kernels", - "etdump_flatcc", -] - -[targets.size_test] -buck_targets = [ - "//test:size_test", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch_core", - "executorch", - "extension_data_loader", -] -# ---------------------------------- binary end ---------------------------------- -# ---------------------------------- MPS start ---------------------------------- -[targets.mps_executor_runner] -buck_targets = [ - "//examples/apple/mps/executor_runner:mps_executor_runner", -] -filters = [ - "(.mm|.cpp)$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch", - "executorch_core", - "extension_evalue_util", - "extension_runner_util", - "extension_threadpool", - "kernels_util_all_deps", - "portable_kernels", -] - -[targets.mps_backend] -buck_targets = [ - "//backends/apple/mps:mps", -] -filters = [ - "(.mm|.cpp)$", -] -deps = [ - "executorch", - "executorch_core", -] - -[targets.mps_schema] -buck_targets = [ - "//backends/apple/mps:mps_schema", -] -filters = [ - ".fbs$", -] - -# ---------------------------------- MPS end ---------------------------------- -# ---------------------------------- XNNPACK start ---------------------------------- - -[targets.xnn_executor_runner] -buck_targets = [ - "//examples/xnnpack:xnn_executor_runner", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch", - "executorch_core", - "extension_evalue_util", - "extension_runner_util", - "extension_threadpool", - "kernels_util_all_deps", - "xnnpack_backend", - "portable_kernels", - "etdump_flatcc", -] - -[targets.xnnpack_backend] -buck_targets = [ - "//backends/xnnpack:xnnpack_backend", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch", - "executorch_core", - "extension_threadpool", -] - -[targets.xnnpack_schema] -buck_targets = [ - "//backends/xnnpack/serialization:xnnpack_flatbuffer_header", -] -filters = [ - ".fbs$", -] -# ---------------------------------- XNNPACK end ---------------------------------- -# ---------------------------------- Vulkan start --------------------------------- -[targets.vulkan_schema] -buck_targets = [ - "//backends/vulkan/serialization:vk_delegate_schema", -] -filters = [ - ".fbs$", -] -# ---------------------------------- Vulkan end ----------------------------------- -# ---------------------------------- LLama start ---------------------------------- -[targets.custom_ops] -buck_targets = [ - "//extension/llm/custom_ops:custom_ops", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "executorch", - "executorch_core", - "optimized_cpublas", - "optimized_kernels", - "extension_threadpool", - "reduce_util", - "xnnpack_backend", -] - -[targets.llama_runner] -buck_targets = [ - "//examples/models/llama/runner:runner", -] -filters = [ - ".cpp$", -] -excludes = [ - "^codegen", -] -deps = [ - "custom_ops", - "executorch", - "executorch_core", - "extension_data_loader", - "extension_flat_tensor", - "extension_llm_runner", - "extension_module", - "extension_tensor", - "extension_threadpool", - "extension_tokenizers", - "kernels_util_all_deps", - "optimized_cpublas", - "portable_kernels", - "quantized_kernels", - "xnnpack_backend", - "optimized_native_cpu_ops", -] -# ---------------------------------- LLama end ---------------------------------- -# ---------------------------------- devtools start ---------------------------------- -[targets.bundled_program] -buck_targets = [ - "//devtools/bundled_program:runtime", -] -filters = [ - ".cpp$", -] -deps = [ - "executorch_core", -] - -[targets.etdump_flatcc] -buck_targets = [ - "//devtools/etdump:etdump_flatcc", -] -filters = [ - ".cpp$", -] -# ---------------------------------- devtools end ---------------------------------- diff --git a/tools/cmake/extract_sources.py b/tools/cmake/extract_sources.py deleted file mode 100755 index 5af0904fdfd..00000000000 --- a/tools/cmake/extract_sources.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import copy -import logging -import os -import re - -from enum import Enum -from typing import Any, List, Optional, Sequence - -from buck_util import Buck2Runner - -try: - import tomllib # Standard in 3.11 and later -except ModuleNotFoundError: - import tomli as tomllib # type: ignore[no-redef] - -"""Extracts source lists from the buck2 build system and writes them to a file. - -The config file is in TOML format and should contains one or more -`[targets.]` entries, along with an optional `[target_base]` entry. - -All of these may have the following lists of strings: -- buck_targets: The list of buck targets that map to ``. -- deps: A list of other `` entries that this target depends on. - Used to prune sources that are provided by those other targets. -- filters: A list of regular expressions. This tool will only emit source files - whose relative paths match all entries. -- excludes: A list of regular expressions. This tool will not emit source files - whose relative paths match any entry. - -The special `[target_base]` entry provides default lists that are inherited by -the `[target.]` entries. When the `[target.]` entry defines -a key that is already present in `[target_base]`, the target-specific entries are -appended to the base list. - -Example config: - - [target_base] - excludes = [ - "^third-party", - ] - - [targets.schema] - buck_targets = [ - "//schema:schema", - ] - filters = [ - ".fbs$", - ] - - [targets.executorch] - buck_targets = [ - "//runtime/executor:program", - ] - deps = [ - "schema", - ] - filters = [ - ".cpp$", - ] -""" - -# Set up logging -logging.basicConfig( - level=logging.INFO, format="%(asctime)s [ExecuTorch] %(levelname)s: %(message)s" -) -logger = logging.getLogger() - - -class Target: - """Parsed [targets.*] entry from the TOML file. - - Can query buck for its list of source files. - """ - - class _InitState(Enum): - UNINITIALIZED = 0 - INITIALIZING = 1 - READY = 2 - - def __init__( - self, - name: str, - target_dict: dict[str, Sequence[str]], - base_dict: Optional[dict] = None, - ) -> None: - self._state: Target._InitState = Target._InitState.UNINITIALIZED - self._sources: frozenset[str] = frozenset() - - self.name = name - # Extend the base lists with the target-specific entries. - self._config = copy.deepcopy(base_dict or {}) - for k, v in target_dict.items(): - if k in self._config: - self._config[k].extend(v) - else: - self._config[k] = v - - def get_sources( - self, graph: "Graph", runner: Buck2Runner, buck_args: Optional[List[str]] - ) -> frozenset[str]: - if buck_args is None: - buck_args = [] - - if self._state == Target._InitState.READY: - return self._sources - # Detect cycles. - assert self._state != Target._InitState.INITIALIZING - - # Assemble the query. - query = "inputs({})".format( - "+".join( - [ - "deps('{}')".format(target) - for target in self._config.get("buck_targets", []) - ] - ) - ) - - # Get the complete list of source files that this target depends on. - # If user doesn't setup their git submodules correctly, this will fail. - # If we hit here, setup.py:check_submodule() should have already run - # but it could be that the submodules are not synced or there's local changes. - try: - sources: set[str] = set(runner.run(["cquery", query] + buck_args)) - except RuntimeError as e: - logger.error( - f"\033[31;1mFailed to query buck for sources. Failed command:\n\n" - f" buck2 cquery {query} {' '.join(buck_args)}\n\n" - "This is likely due " - "to missing git submodules or outdated CMake cache. " - "Please run the following before retry:\033[0m\n\n" - " \033[32;1m./install_executorch.sh --clean\033[0m\n" - " \033[32;1mgit submodule sync\033[0m\n" - " \033[32;1mgit submodule update --init\033[0m\n" - ) - raise e - - # Keep entries that match all of the filters. - filters = [re.compile(p) for p in self._config.get("filters", [])] - sources = {s for s in sources if all(p.search(s) for p in filters)} - - # Remove entries that match any of the excludes. - excludes = [re.compile(p) for p in self._config.get("excludes", [])] - sources = {s for s in sources if not any(p.search(s) for p in excludes)} - - # The buck query will give us the complete list of sources that this - # target depends on, but that list includes sources that are owned by - # its deps. Remove entries that are already covered by the transitive - # set of dependencies. - for dep in self._config.get("deps", []): - sources.difference_update( - graph.by_name[dep].get_sources(graph, runner, buck_args) - ) - - self._sources = frozenset(sources) - self._state = Target._InitState.READY - return self._sources - - -class Graph: - """Graph of targets.""" - - def __init__(self, config_dict: dict[str, Any]) -> None: - base = config_dict.get("target_base", {}) - targets = config_dict.get("targets", {}) - - self.by_name = {} - for k, v in targets.items(): - self.by_name[k] = Target(k, v, base) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Extracts deps from the buck2 build system", - ) - parser.add_argument( - "--buck2", - default="buck2", - help="'buck2' command to use", - ) - parser.add_argument( - "--config", - metavar="config.toml", - required=True, - help="Path to the input TOML configuration file", - ) - parser.add_argument( - "--format", - default="cmake", - choices=["cmake"], - help="Format to generate.", - ) - parser.add_argument( - "--out", - metavar="file", - help="Path to the file to generate.", - ) - parser.add_argument( - "--target-platforms", help="--target-platforms to pass to buck cquery, if any." - ) - return parser.parse_args() - - -def generate_cmake(target_to_srcs: dict[str, list[str]]) -> bytes: - lines: list[str] = [] - lines.append("# @" + f"generated by {os.path.basename(__file__)}") - for target, srcs in target_to_srcs.items(): - lines.append("") - lines.append(f"set(_{target}__srcs") - for src in srcs: - lines.append(f" {src}") - lines.append(")") - return "\n".join(lines).encode("utf-8") - - -def main(): - args = parse_args() - - # Load and parse the TOML configuration - with open(args.config, mode="rb") as fp: - config_dict = tomllib.load(fp) - graph = Graph(config_dict) - - # Run the queries and get the lists of source files. - target_to_srcs: dict[str, list[str]] = {} - runner: Buck2Runner = Buck2Runner(args.buck2) - buck_args = [] - if args.target_platforms: - buck_args = ["--target-platforms"] - buck_args.append(args.target_platforms) - for name, target in graph.by_name.items(): - target_to_srcs[name] = sorted(target.get_sources(graph, runner, buck_args)) - - # Generate the requested format. - output: bytes - if args.format == "cmake": - output = generate_cmake(target_to_srcs) - else: - raise ValueError("Unknown format: {}".format(args.format)) - - # Write the output. - with open(args.out, "wb") as fp: - fp.write(output) - - -if __name__ == "__main__": - main()