diff --git a/.bazelrc b/.bazelrc index 801b963ad5..9c7ad71323 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,11 +1,142 @@ -# For bazel-in-bazel testing -# Trick bazel into treating BUILD files under examples/* as being regular files +# For bazel-in-bazel testing and ignoring sub-workspaces for ... patterns. +# +# Trick Bazel into treating BUILD files under examples/* as being regular files # This lets us glob() up all the files inside the examples to make them inputs to tests -# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute -# `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg +# `tools/update_deleted_packages.sh` +# GENERATED_DELETED_PACKAGES_START +common --deleted_packages=examples/build_file_generation +common --deleted_packages=examples/build_file_generation/random_number_generator +common --deleted_packages=examples/bzlmod +common --deleted_packages=examples/bzlmod/entry_points +common --deleted_packages=examples/bzlmod/entry_points/tests +common --deleted_packages=examples/bzlmod/libs/my_lib +common --deleted_packages=examples/bzlmod/other_module +common --deleted_packages=examples/bzlmod/other_module/other_module/pkg +common --deleted_packages=examples/bzlmod/patches +common --deleted_packages=examples/bzlmod/py_proto_library +common --deleted_packages=examples/bzlmod/py_proto_library/example.com/another_proto +common --deleted_packages=examples/bzlmod/py_proto_library/example.com/proto +common --deleted_packages=examples/bzlmod/py_proto_library/foo_external +common --deleted_packages=examples/bzlmod/runfiles +common --deleted_packages=examples/bzlmod/tests +common --deleted_packages=examples/bzlmod/tests/other_module +common --deleted_packages=examples/bzlmod/whl_mods +common --deleted_packages=examples/bzlmod_build_file_generation +common --deleted_packages=examples/bzlmod_build_file_generation/other_module +common --deleted_packages=examples/bzlmod_build_file_generation/other_module/other_module/pkg +common --deleted_packages=examples/bzlmod_build_file_generation/runfiles +common --deleted_packages=examples/multi_python_versions +common --deleted_packages=examples/multi_python_versions/libs/my_lib +common --deleted_packages=examples/multi_python_versions/requirements +common --deleted_packages=examples/multi_python_versions/tests +common --deleted_packages=examples/pip_parse +common --deleted_packages=examples/pip_parse_vendored +common --deleted_packages=examples/pip_repository_annotations +common --deleted_packages=examples/py_proto_library +common --deleted_packages=examples/py_proto_library/example.com/another_proto +common --deleted_packages=examples/py_proto_library/example.com/proto +common --deleted_packages=gazelle +common --deleted_packages=gazelle/docs +common --deleted_packages=gazelle/manifest +common --deleted_packages=gazelle/manifest/generate +common --deleted_packages=gazelle/manifest/hasher +common --deleted_packages=gazelle/manifest/test +common --deleted_packages=gazelle/modules_mapping +common --deleted_packages=gazelle/python +common --deleted_packages=gazelle/python/private +common --deleted_packages=gazelle/python/testdata/add_type_stub_packages +common --deleted_packages=gazelle/python/testdata/annotation_include_dep +common --deleted_packages=gazelle/python/testdata/annotation_include_pytest_conftest +common --deleted_packages=gazelle/python/testdata/binary_without_entrypoint +common --deleted_packages=gazelle/python/testdata/binary_without_entrypoint_per_file_generation +common --deleted_packages=gazelle/python/testdata/binary_without_entrypoint_per_file_generation_partial_update +common --deleted_packages=gazelle/python/testdata/clear_out_deps +common --deleted_packages=gazelle/python/testdata/dependency_resolution_order +common --deleted_packages=gazelle/python/testdata/different_packages_in_same_namespace +common --deleted_packages=gazelle/python/testdata/directive_python_default_visibility +common --deleted_packages=gazelle/python/testdata/directive_python_generate_proto +common --deleted_packages=gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf +common --deleted_packages=gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed +common --deleted_packages=gazelle/python/testdata/directive_python_label_convention +common --deleted_packages=gazelle/python/testdata/directive_python_label_normalization +common --deleted_packages=gazelle/python/testdata/directive_python_proto_naming_convention +common --deleted_packages=gazelle/python/testdata/directive_python_test_file_pattern +common --deleted_packages=gazelle/python/testdata/directive_python_test_file_pattern_bad_glob +common --deleted_packages=gazelle/python/testdata/directive_python_test_file_pattern_no_value +common --deleted_packages=gazelle/python/testdata/directive_python_visibility +common --deleted_packages=gazelle/python/testdata/disable_import_statements_validation +common --deleted_packages=gazelle/python/testdata/dont_ignore_setup +common --deleted_packages=gazelle/python/testdata/dont_rename_target +common --deleted_packages=gazelle/python/testdata/file_name_matches_import_statement +common --deleted_packages=gazelle/python/testdata/first_party_dependencies +common --deleted_packages=gazelle/python/testdata/first_party_file_and_directory_modules +common --deleted_packages=gazelle/python/testdata/from_imports +common --deleted_packages=gazelle/python/testdata/generated_test_entrypoint +common --deleted_packages=gazelle/python/testdata/ignored_invalid_imported_module +common --deleted_packages=gazelle/python/testdata/invalid_annotation_exclude +common --deleted_packages=gazelle/python/testdata/invalid_annotation_include_dep +common --deleted_packages=gazelle/python/testdata/invalid_imported_module +common --deleted_packages=gazelle/python/testdata/monorepo +common --deleted_packages=gazelle/python/testdata/multiple_tests +common --deleted_packages=gazelle/python/testdata/naming_convention +common --deleted_packages=gazelle/python/testdata/naming_convention_binary_fail +common --deleted_packages=gazelle/python/testdata/naming_convention_library_fail +common --deleted_packages=gazelle/python/testdata/naming_convention_test_fail +common --deleted_packages=gazelle/python/testdata/per_file +common --deleted_packages=gazelle/python/testdata/per_file_non_empty_init +common --deleted_packages=gazelle/python/testdata/per_file_subdirs +common --deleted_packages=gazelle/python/testdata/per_package_test_target_without_entry_point +common --deleted_packages=gazelle/python/testdata/project_generation_mode +common --deleted_packages=gazelle/python/testdata/project_generation_mode_with_test_entrypoint +common --deleted_packages=gazelle/python/testdata/project_generation_mode_with_tests +common --deleted_packages=gazelle/python/testdata/py312_syntax +common --deleted_packages=gazelle/python/testdata/python_ignore_dependencies_directive +common --deleted_packages=gazelle/python/testdata/python_ignore_files_directive +common --deleted_packages=gazelle/python/testdata/python_target_with_test_in_name +common --deleted_packages=gazelle/python/testdata/relative_imports_package_mode +common --deleted_packages=gazelle/python/testdata/relative_imports_project_mode +common --deleted_packages=gazelle/python/testdata/remove_invalid_library +common --deleted_packages=gazelle/python/testdata/respect_kind_mapping +common --deleted_packages=gazelle/python/testdata/sibling_imports +common --deleted_packages=gazelle/python/testdata/sibling_imports_disabled +common --deleted_packages=gazelle/python/testdata/sibling_imports_disabled_file_mode +common --deleted_packages=gazelle/python/testdata/simple_binary +common --deleted_packages=gazelle/python/testdata/simple_binary_with_library +common --deleted_packages=gazelle/python/testdata/simple_library +common --deleted_packages=gazelle/python/testdata/simple_library_without_init +common --deleted_packages=gazelle/python/testdata/simple_test +common --deleted_packages=gazelle/python/testdata/simple_test_with_conftest +common --deleted_packages=gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled +common --deleted_packages=gazelle/python/testdata/subdir_sources +common --deleted_packages=gazelle/python/testdata/type_checking_imports +common --deleted_packages=gazelle/python/testdata/type_checking_imports_across_packages +common --deleted_packages=gazelle/python/testdata/type_checking_imports_disabled +common --deleted_packages=gazelle/python/testdata/type_checking_imports_package +common --deleted_packages=gazelle/python/testdata/type_checking_imports_project +common --deleted_packages=gazelle/python/testdata/with_nested_import_statements +common --deleted_packages=gazelle/python/testdata/with_std_requirements +common --deleted_packages=gazelle/python/testdata/with_third_party_requirements +common --deleted_packages=gazelle/python/testdata/with_third_party_requirements_from_imports +common --deleted_packages=gazelle/pythonconfig +common --deleted_packages=tests/integration/compile_pip_requirements +common --deleted_packages=tests/integration/compile_pip_requirements_test_from_external_repo +common --deleted_packages=tests/integration/custom_commands +common --deleted_packages=tests/integration/ignore_root_user_error +common --deleted_packages=tests/integration/ignore_root_user_error/submodule +common --deleted_packages=tests/integration/local_toolchains +common --deleted_packages=tests/integration/pip_parse +common --deleted_packages=tests/integration/pip_parse/empty +common --deleted_packages=tests/integration/py_cc_toolchain_registered +common --deleted_packages=tests/modules/another_module +common --deleted_packages=tests/modules/other +common --deleted_packages=tests/modules/other/nspkg_delta +common --deleted_packages=tests/modules/other/nspkg_gamma +common --deleted_packages=tests/modules/other/nspkg_single +common --deleted_packages=tests/modules/other/simple_v1 +common --deleted_packages=tests/modules/other/simple_v2 +common --deleted_packages=tests/modules/other/with_external_data +# GENERATED_DELETED_PACKAGES_END test --test_output=errors diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 67a02fc6c0..2d89876673 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,8 +46,6 @@ repos: - id: update-deleted-packages name: Update deleted packages language: system - # 7.x is necessary until https://github.com/bazel-contrib/rules_bazel_integration_test/pull/414 - # is merged and released - entry: env USE_BAZEL_VERSION=7.x bazel run @rules_bazel_integration_test//tools:update_deleted_packages + entry: tools/update_deleted_packages.sh files: ^((examples|tests)/.*/(MODULE.bazel|WORKSPACE|WORKSPACE.bzlmod|BUILD.bazel)|.bazelrc)$ pass_filenames: false diff --git a/tools/update_deleted_packages.sh b/tools/update_deleted_packages.sh index 17e33d182a..8f077c53ed 100755 --- a/tools/update_deleted_packages.sh +++ b/tools/update_deleted_packages.sh @@ -13,27 +13,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -# For integration tests, we want to be able to glob() up the sources inside a nested package -# See explanation in .bazelrc -# -# This script ensures that we only delete subtrees that have something a file -# signifying a new bazel workspace, whether it be bzlmod or classic. Generic -# algorithm: -# 1. Get all directories where a WORKSPACE or MODULE.bazel exists. -# 2. For each of the directories, get all directories that contains a BUILD.bazel file. -# 3. Sort and remove duplicates. - -set -euxo pipefail - -DIR="$(dirname $0)/.." -cd $DIR - -# The sed -i.bak pattern is compatible between macos and linux -sed -i.bak "/^[^#].*--deleted_packages/s#=.*#=$(\ - find examples/*/* tests/*/* \( -name WORKSPACE -or -name MODULE.bazel \) | - xargs -n 1 dirname | - xargs -n 1 -I{} find {} \( -name BUILD -or -name BUILD.bazel \) | - xargs -n 1 dirname | - sort -u | - paste -sd, -\ -)#" $DIR/.bazelrc && rm .bazelrc.bak +# This script finds Bazel sub-workspaces and marks them, and all their BUILD +# containing directories, as ignored by the root level bazel project, via the +# `--deleted_packages` flag. + +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +ROOT_DIR="$DIR/.." +BAZELRC="$ROOT_DIR/.bazelrc" + +# Find all packages in sub-workspaces +found_packages() { + cd "$ROOT_DIR" + ( + find . -mindepth 2 \( -name WORKSPACE -o -name MODULE.bazel \) | + while read -r marker; do + workspace_dir="$(dirname "$marker")" + echo "$workspace_dir" + find "$workspace_dir" \( -name BUILD -o -name BUILD.bazel \) -exec dirname {} \; + done + ) | sed 's#^\./##' +} + +# Update .bazelrc +update_bazelrc() { + local packages + # Set LC_COLLATE so that "/" sorts before "_", which makes + # for slightly nicer ordering with file paths + packages=$(found_packages | LC_COLLATE=C sort | uniq) + + local start_marker="# GENERATED_DELETED_PACKAGES_START" + local end_marker="# GENERATED_DELETED_PACKAGES_END" + + # Create a temporary file + local tmpfile + tmpfile=$(mktemp) + + # Write the content before the start marker + sed "/$start_marker/q" "$BAZELRC" > "$tmpfile" + + # Write the generated packages + for pkg in $packages; do + echo "common --deleted_packages=$pkg" >> "$tmpfile" + done + echo "$end_marker" >> "$tmpfile" + + # Write the content after the end marker + sed "1,/$end_marker/d" "$BAZELRC" >> "$tmpfile" + + # Replace the original file + mv "$tmpfile" "$BAZELRC" +} + +update_bazelrc