Fix buck query //extension/flat_tensor: in OSS

.ci/scripts/unittest-buck2.sh

@@ -11,9 +11,10 @@ set -eux
 # TODO: can't query //kernels/prim_ops because of non-buckified stuff in OSS.
 buck2 query "//backends/apple/... + //backends/example/... + \
 //backends/mediatek/... + //backends/transforms/... + \
-//backends/xnnpack/... + //configurations/... + //kernels/aten/... + \
-//kernels/optimized/... + //kernels/portable/... + //kernels/quantized/... + \
-//kernels/test/... + //runtime/... + //schema/... + //test/... + //util/..."
+//backends/xnnpack/... + //configurations/... + //extension/flat_tensor: + \
+//kernels/aten/... + //kernels/optimized/... + //kernels/portable/... + \
+//kernels/quantized/... + //kernels/test/... + //runtime/... + //schema/... \
++ //test/... + //util/..."
 
 # TODO: optimized ops are unbuildable because they now use ATen; put
 # them back after we can use PyTorch in OSS buck.

extension/flat_tensor/targets.bzl

@@ -1,7 +1,7 @@
-load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
+load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_aten_mode_options", "runtime")
 
 def define_common_targets():
-    for aten_mode in [True, False]:
+    for aten_mode in get_aten_mode_options():
         aten_suffix = "_aten" if aten_mode else ""
         runtime.cxx_library(
             name = "flat_tensor_data_map" + aten_suffix,

pyproject.toml

@@ -4,7 +4,6 @@ requires = [
   "pip>=23",  # For building the pip package.
   "pyyaml",  # Imported by the kernel codegen tools.
   "setuptools>=63",  # For building the pip package contents.
-  "tomli",  # Imported by extract_sources.py when using python < 3.11.
   "wheel",  # For building the pip package archive.
   "zstd",  # Imported by resolve_buck.py.
   "certifi",  # Imported by resolve_buck.py.

requirements-dev.txt

@@ -4,7 +4,6 @@ cmake>=3.29, <4.0.0  # For building binary targets in the wheel.
 pip>=23  # For building the pip package.
 pyyaml  # Imported by the kernel codegen tools.
 setuptools>=63  # For building the pip package contents.
-tomli  # Imported by extract_sources.py when using python < 3.11.
 wheel  # For building the pip package archive.
 zstd  # Imported by resolve_buck.py.
 certifi  # Imported by resolve_buck.py.

shim_et/xplat/executorch/build/build_variables.bzl

@@ -290,11 +290,6 @@ QUANTIZED_KERNELS_SRCS = [
     "kernels/quantized/cpu/op_quantize.cpp",
 ]
 
-PROGRAM_SCHEMA_SRCS = [
-    "schema/program.fbs",
-    "schema/scalar_type.fbs",
-]
-
 OPTIMIZED_CPUBLAS_SRCS = [
     "kernels/optimized/blas/BlasKernel.cpp",
     "kernels/optimized/blas/CPUBlas.cpp",
@@ -375,27 +370,14 @@ THREADPOOL_SRCS = [
 EXTENSION_THREADPOOL_SRCS = ["extension/threadpool/" + x for x in THREADPOOL_SRCS]
 
 EXTENSION_TRAINING_SRCS = [
-    "extension/data_loader/file_data_loader.cpp",
-    "extension/data_loader/mmap_data_loader.cpp",
-    "extension/flat_tensor/flat_tensor_data_map.cpp",
-    "extension/flat_tensor/serialize/flat_tensor_header.cpp",
-    "extension/module/module.cpp",
     "extension/training/module/training_module.cpp",
     "extension/training/optimizer/sgd.cpp",
 ]
 
 TRAIN_XOR_SRCS = [
-    "extension/data_loader/file_data_loader.cpp",
-    "extension/data_loader/mmap_data_loader.cpp",
-    "extension/flat_tensor/flat_tensor_data_map.cpp",
-    "extension/flat_tensor/serialize/flat_tensor_header.cpp",
+    # REVIEW: removing this breaks the build; where is it supposed to come from?
     "extension/flat_tensor/serialize/serialize.cpp",
-    "extension/module/module.cpp",
-    "extension/tensor/tensor_ptr.cpp",
-    "extension/tensor/tensor_ptr_maker.cpp",
     "extension/training/examples/XOR/train.cpp",
-    "extension/training/module/training_module.cpp",
-    "extension/training/optimizer/sgd.cpp",
 ]
 
 EXECUTOR_RUNNER_SRCS = [

tools/cmake/Codegen.cmake

@@ -386,7 +386,6 @@ function(executorch_load_build_variables)
       KERNELS_UTIL_ALL_DEPS_SRCS
       OPTIMIZED_KERNELS_SRCS
       QUANTIZED_KERNELS_SRCS
-      PROGRAM_SCHEMA_SRCS
       OPTIMIZED_CPUBLAS_SRCS
       OPTIMIZED_NATIVE_CPU_OPS_SRCS
       TEST_BACKEND_COMPILER_LIB_SRCS
@@ -419,7 +418,6 @@ function(executorch_load_build_variables)
       _kernels_util_all_deps__srcs
       _optimized_kernels__srcs
       _quantized_kernels__srcs
-      _program_schema__srcs
       _optimized_cpublas__srcs
       _optimized_native_cpu_ops__srcs
       _test_backend_compiler_lib__srcs

tools/cmake/Utils.cmake

@@ -67,58 +67,6 @@ function(target_link_options_gc_sections target_name)
   endif()
 endfunction()
 
-# Extract source files based on toml config. This is useful to keep buck2 and
-# cmake aligned. Do not regenerate if file exists.
-function(extract_sources sources_file)
-  if(EXISTS "${sources_file}")
-    message(STATUS "executorch: Using source file list ${sources_file}")
-  else()
-    # A file wasn't generated. Run a script to extract the source lists from the
-    # buck2 build system and write them to a file we can include.
-    #
-    # NOTE: This will only happen once during cmake setup, so it will not re-run
-    # if the buck2 targets change.
-    message(STATUS "executorch: Generating source file list ${sources_file}")
-    if(EXECUTORCH_ROOT)
-      set(executorch_root ${EXECUTORCH_ROOT})
-    else()
-      set(executorch_root ${CMAKE_CURRENT_SOURCE_DIR})
-    endif()
-
-    if(ANDROID_ABI)
-      if("${ANDROID_ABI}" STREQUAL "arm64-v8a")
-        set(target_platforms_arg "--target-platforms=shim_et//:android-arm64")
-      elseif("${ANDROID_ABI}" STREQUAL "x86_64")
-        set(target_platforms_arg "--target-platforms=shim_et//:android-x86_64")
-      else()
-        message(
-          FATAL_ERROR
-            "Unsupported ANDROID_ABI setting ${ANDROID_ABI}. Please add it here!"
-        )
-      endif()
-    endif()
-    execute_process(
-      COMMAND
-        ${PYTHON_EXECUTABLE} ${executorch_root}/tools/cmake/extract_sources.py
-        --config=${executorch_root}/tools/cmake/cmake_deps.toml
-        --out=${sources_file} --buck2=${BUCK2} ${target_platforms_arg}
-      OUTPUT_VARIABLE gen_srcs_output
-      ERROR_VARIABLE gen_srcs_error
-      RESULT_VARIABLE gen_srcs_exit_code
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    )
-
-    if(NOT gen_srcs_exit_code EQUAL 0)
-      message("Error while generating ${sources_file}. "
-              "Exit code: ${gen_srcs_exit_code}"
-      )
-      message("Output:\n${gen_srcs_output}")
-      message("Error:\n${gen_srcs_error}")
-      message(FATAL_ERROR "executorch: source list generation failed")
-    endif()
-  endif()
-endfunction()
-
 # Sets the value of the PYTHON_EXECUTABLE variable to 'python' if in an active
 # (non-base) conda environment, and 'python3' otherwise. This maintains
 # backwards compatibility for non-conda users and avoids conda users needing to