From d3c0a1a5399bde23d5dc9f428e6d12c2bc6002cf Mon Sep 17 00:00:00 2001 From: Vivek Dasari Date: Tue, 24 Jun 2025 17:08:07 -0700 Subject: [PATCH 01/12] Respect python_root when resolving imports; treat sibling imports outside of root as 3rd party --- gazelle/python/kinds.go | 2 +- gazelle/python/target.go | 10 ------- .../import_resolution_hierarchy/BUILD.in | 0 .../import_resolution_hierarchy/BUILD.out | 7 +++++ .../import_resolution_hierarchy/MODULE.bazel | 6 ++++ .../import_resolution_hierarchy/README.md | 29 +++++++++++++++++++ .../import_resolution_hierarchy/WORKSPACE | 1 + .../import_resolution_hierarchy/__init__.py | 0 .../import_resolution_hierarchy/a/BUILD.in | 2 ++ .../import_resolution_hierarchy/a/BUILD.out | 17 +++++++++++ .../import_resolution_hierarchy/a/__init__.py | 0 .../import_resolution_hierarchy/a/b/BUILD.in | 0 .../import_resolution_hierarchy/a/b/BUILD.out | 16 ++++++++++ .../a/b/__init__.py | 0 .../import_resolution_hierarchy/a/b/bar.py | 15 ++++++++++ .../import_resolution_hierarchy/a/b/foo.py | 15 ++++++++++ .../import_resolution_hierarchy/a/bar.py | 15 ++++++++++ .../import_resolution_hierarchy/a/foo.py | 15 ++++++++++ .../import_resolution_hierarchy/test.yaml | 15 ++++++++++ 19 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/BUILD.in create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/BUILD.out create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/MODULE.bazel create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/README.md create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/WORKSPACE create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/__init__.py create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/a/BUILD.in create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/a/BUILD.out create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/a/__init__.py create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/a/b/BUILD.in create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/a/b/BUILD.out create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/a/b/__init__.py create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/a/b/bar.py create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/a/b/foo.py create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/a/bar.py create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/a/foo.py create mode 100644 gazelle/python/testdata/import_resolution_hierarchy/test.yaml diff --git a/gazelle/python/kinds.go b/gazelle/python/kinds.go index ff3f6ce829..83da53edbd 100644 --- a/gazelle/python/kinds.go +++ b/gazelle/python/kinds.go @@ -32,7 +32,7 @@ func (*Python) Kinds() map[string]rule.KindInfo { var pyKinds = map[string]rule.KindInfo{ pyBinaryKind: { - MatchAny: false, + MatchAny: false, MatchAttrs: []string{"srcs"}, NonEmptyAttrs: map[string]bool{ "deps": true, diff --git a/gazelle/python/target.go b/gazelle/python/target.go index 06b653d915..3b7f464335 100644 --- a/gazelle/python/target.go +++ b/gazelle/python/target.go @@ -71,16 +71,6 @@ func (t *targetBuilder) addSrcs(srcs *treeset.Set) *targetBuilder { // addModuleDependency adds a single module dep to the target. func (t *targetBuilder) addModuleDependency(dep Module) *targetBuilder { - fileName := dep.Name + ".py" - if dep.From != "" { - fileName = dep.From + ".py" - } - if t.siblingSrcs.Contains(fileName) && fileName != filepath.Base(dep.Filepath) { - // importing another module from the same package, converting to absolute imports to make - // dependency resolution easier - dep.Name = importSpecFromSrc(t.pythonProjectRoot, t.bzlPackage, fileName).Imp - } - addModuleToTreeSet(t.deps, dep) return t } diff --git a/gazelle/python/testdata/import_resolution_hierarchy/BUILD.in b/gazelle/python/testdata/import_resolution_hierarchy/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/import_resolution_hierarchy/BUILD.out b/gazelle/python/testdata/import_resolution_hierarchy/BUILD.out new file mode 100644 index 0000000000..61a7bfdd56 --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/BUILD.out @@ -0,0 +1,7 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "import_resolution_hierarchy", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/import_resolution_hierarchy/MODULE.bazel b/gazelle/python/testdata/import_resolution_hierarchy/MODULE.bazel new file mode 100644 index 0000000000..00bb18361f --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/MODULE.bazel @@ -0,0 +1,6 @@ +############################################################################### +# Bazel now uses Bzlmod by default to manage external dependencies. +# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel. +# +# For more details, please check https://github.com/bazelbuild/bazel/issues/18958 +############################################################################### diff --git a/gazelle/python/testdata/import_resolution_hierarchy/README.md b/gazelle/python/testdata/import_resolution_hierarchy/README.md new file mode 100644 index 0000000000..49f78975fc --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/README.md @@ -0,0 +1,29 @@ +# Python Import Resolution Hierarchy + +This test case verifies that Python imports are resolved correctly following Python's module search path semantics, especially when there are multiple files with the same name in different directories. + +## Test Scenario + +The directory structure is: +``` +a/ +├── bar.py # Module at project level +├── foo.py # Imports "bar" - should resolve to a/bar.py +└── b/ + ├── bar.py # Local module in subdirectory + └── foo.py # Imports "bar" - should resolve to a/bar.py (NOT a/b/bar.py) +``` + +## Expected Behavior + +When `a/b/foo.py` contains `import bar`, it should resolve to `//a:bar` (the module at the parent level), not `//a/b:bar` (the local module), following Python's import resolution semantics where imports are resolved from the project root. + +## Bug Fixed + +This test case was created to verify the fix for a bug where `import bar` from `a/b/foo.py` was incorrectly resolving to `:bar` (local `a/b/bar.py`) instead of `//a:bar` (parent-level `a/bar.py`). + +## Generation Mode + +Uses per-file generation mode where each `.py` file generates its own `py_library` target. + +`__init__.py` files are left empty so no target is generated for them. diff --git a/gazelle/python/testdata/import_resolution_hierarchy/WORKSPACE b/gazelle/python/testdata/import_resolution_hierarchy/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/import_resolution_hierarchy/__init__.py b/gazelle/python/testdata/import_resolution_hierarchy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/import_resolution_hierarchy/a/BUILD.in b/gazelle/python/testdata/import_resolution_hierarchy/a/BUILD.in new file mode 100644 index 0000000000..2ee1e0d435 --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/a/BUILD.in @@ -0,0 +1,2 @@ +# gazelle:python_generation_mode file +# gazelle:python_root diff --git a/gazelle/python/testdata/import_resolution_hierarchy/a/BUILD.out b/gazelle/python/testdata/import_resolution_hierarchy/a/BUILD.out new file mode 100644 index 0000000000..c9274d8585 --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/a/BUILD.out @@ -0,0 +1,17 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode file +# gazelle:python_root + +py_library( + name = "bar", + srcs = ["bar.py"], + visibility = ["//a:__subpackages__"], +) + +py_library( + name = "foo", + srcs = ["foo.py"], + visibility = ["//a:__subpackages__"], + deps = [":bar"], +) diff --git a/gazelle/python/testdata/import_resolution_hierarchy/a/__init__.py b/gazelle/python/testdata/import_resolution_hierarchy/a/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/import_resolution_hierarchy/a/b/BUILD.in b/gazelle/python/testdata/import_resolution_hierarchy/a/b/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/import_resolution_hierarchy/a/b/BUILD.out b/gazelle/python/testdata/import_resolution_hierarchy/a/b/BUILD.out new file mode 100644 index 0000000000..2001814eb0 --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/a/b/BUILD.out @@ -0,0 +1,16 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "bar", + srcs = ["bar.py"], + imports = [".."], + visibility = ["//a:__subpackages__"], +) + +py_library( + name = "foo", + srcs = ["foo.py"], + imports = [".."], + visibility = ["//a:__subpackages__"], + deps = ["//a:bar"], +) diff --git a/gazelle/python/testdata/import_resolution_hierarchy/a/b/__init__.py b/gazelle/python/testdata/import_resolution_hierarchy/a/b/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/import_resolution_hierarchy/a/b/bar.py b/gazelle/python/testdata/import_resolution_hierarchy/a/b/bar.py new file mode 100644 index 0000000000..3bf11bfe19 --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/a/b/bar.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For test purposes only. \ No newline at end of file diff --git a/gazelle/python/testdata/import_resolution_hierarchy/a/b/foo.py b/gazelle/python/testdata/import_resolution_hierarchy/a/b/foo.py new file mode 100644 index 0000000000..c000990002 --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/a/b/foo.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import bar diff --git a/gazelle/python/testdata/import_resolution_hierarchy/a/bar.py b/gazelle/python/testdata/import_resolution_hierarchy/a/bar.py new file mode 100644 index 0000000000..730755995d --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/a/bar.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For test purposes only. diff --git a/gazelle/python/testdata/import_resolution_hierarchy/a/foo.py b/gazelle/python/testdata/import_resolution_hierarchy/a/foo.py new file mode 100644 index 0000000000..c000990002 --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/a/foo.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import bar diff --git a/gazelle/python/testdata/import_resolution_hierarchy/test.yaml b/gazelle/python/testdata/import_resolution_hierarchy/test.yaml new file mode 100644 index 0000000000..fcea77710f --- /dev/null +++ b/gazelle/python/testdata/import_resolution_hierarchy/test.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- From 25e758de39b95bf6ea59a19db74b84962852640b Mon Sep 17 00:00:00 2001 From: Vivek Dasari Date: Thu, 26 Jun 2025 16:21:32 -0700 Subject: [PATCH 02/12] fix all broken tests --- gazelle/python/generate.go | 3 ++- .../subpkg/module1_test.py | 4 +-- .../annotation_include_dep/subpkg/module2.py | 4 +-- .../naming_convention/dont_rename/__main__.py | 2 +- .../naming_convention/dont_rename/__test__.py | 2 +- .../resolve_conflict/__main__.py | 2 +- .../resolve_conflict/__test__.py | 2 +- .../python/testdata/sibling_imports/README.md | 3 --- .../python/testdata/sibling_imports/WORKSPACE | 1 - .../testdata/sibling_imports/pkg/BUILD.in | 0 .../testdata/sibling_imports/pkg/BUILD.out | 26 ------------------- .../testdata/sibling_imports/pkg/__init__.py | 0 .../python/testdata/sibling_imports/pkg/a.py | 0 .../python/testdata/sibling_imports/pkg/b.py | 2 -- .../testdata/sibling_imports/pkg/test_util.py | 0 .../testdata/sibling_imports/pkg/unit_test.py | 3 --- .../python/testdata/sibling_imports/test.yaml | 1 - .../simple_test_with_conftest/bar/__init__.py | 2 +- .../simple_test_with_conftest/bar/__test__.py | 2 +- .../subdir_sources/one/two/__init__.py | 2 +- 20 files changed, 12 insertions(+), 49 deletions(-) delete mode 100644 gazelle/python/testdata/sibling_imports/README.md delete mode 100644 gazelle/python/testdata/sibling_imports/WORKSPACE delete mode 100644 gazelle/python/testdata/sibling_imports/pkg/BUILD.in delete mode 100644 gazelle/python/testdata/sibling_imports/pkg/BUILD.out delete mode 100644 gazelle/python/testdata/sibling_imports/pkg/__init__.py delete mode 100644 gazelle/python/testdata/sibling_imports/pkg/a.py delete mode 100644 gazelle/python/testdata/sibling_imports/pkg/b.py delete mode 100644 gazelle/python/testdata/sibling_imports/pkg/test_util.py delete mode 100644 gazelle/python/testdata/sibling_imports/pkg/unit_test.py delete mode 100644 gazelle/python/testdata/sibling_imports/test.yaml diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index c1edec4731..1a1f477a59 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -466,7 +466,8 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes for _, pyTestTarget := range pyTestTargets { if conftest != nil { - pyTestTarget.addModuleDependency(Module{Name: strings.TrimSuffix(conftestFilename, ".py")}) + // Add conftest as a local dependency + pyTestTarget.addResolvedDependency(":" + conftestTargetname) } pyTest := pyTestTarget.build() diff --git a/gazelle/python/testdata/annotation_include_dep/subpkg/module1_test.py b/gazelle/python/testdata/annotation_include_dep/subpkg/module1_test.py index 087763a693..5e0a9bcafa 100644 --- a/gazelle/python/testdata/annotation_include_dep/subpkg/module1_test.py +++ b/gazelle/python/testdata/annotation_include_dep/subpkg/module1_test.py @@ -1,5 +1,3 @@ # gazelle:include_dep //:bagel_from_include_dep_in_module1_test -import module1 - -del module1 +import subpkg.module1 diff --git a/gazelle/python/testdata/annotation_include_dep/subpkg/module2.py b/gazelle/python/testdata/annotation_include_dep/subpkg/module2.py index dabeb6794a..0d395662d3 100644 --- a/gazelle/python/testdata/annotation_include_dep/subpkg/module2.py +++ b/gazelle/python/testdata/annotation_include_dep/subpkg/module2.py @@ -1,4 +1,4 @@ # gazelle:include_dep //other/thing:from_include_dep_in_module2 -import module1 +import subpkg.module1 -del module1 +del subpkg.module1 diff --git a/gazelle/python/testdata/naming_convention/dont_rename/__main__.py b/gazelle/python/testdata/naming_convention/dont_rename/__main__.py index 97955897bf..bb4982f8b2 100644 --- a/gazelle/python/testdata/naming_convention/dont_rename/__main__.py +++ b/gazelle/python/testdata/naming_convention/dont_rename/__main__.py @@ -13,4 +13,4 @@ # limitations under the License. # For test purposes only. -import __init__ +import dont_rename.__init__ diff --git a/gazelle/python/testdata/naming_convention/dont_rename/__test__.py b/gazelle/python/testdata/naming_convention/dont_rename/__test__.py index 97955897bf..bb4982f8b2 100644 --- a/gazelle/python/testdata/naming_convention/dont_rename/__test__.py +++ b/gazelle/python/testdata/naming_convention/dont_rename/__test__.py @@ -13,4 +13,4 @@ # limitations under the License. # For test purposes only. -import __init__ +import dont_rename.__init__ diff --git a/gazelle/python/testdata/naming_convention/resolve_conflict/__main__.py b/gazelle/python/testdata/naming_convention/resolve_conflict/__main__.py index 97955897bf..badff84a9e 100644 --- a/gazelle/python/testdata/naming_convention/resolve_conflict/__main__.py +++ b/gazelle/python/testdata/naming_convention/resolve_conflict/__main__.py @@ -13,4 +13,4 @@ # limitations under the License. # For test purposes only. -import __init__ +import resolve_conflict.__init__ diff --git a/gazelle/python/testdata/naming_convention/resolve_conflict/__test__.py b/gazelle/python/testdata/naming_convention/resolve_conflict/__test__.py index 97955897bf..badff84a9e 100644 --- a/gazelle/python/testdata/naming_convention/resolve_conflict/__test__.py +++ b/gazelle/python/testdata/naming_convention/resolve_conflict/__test__.py @@ -13,4 +13,4 @@ # limitations under the License. # For test purposes only. -import __init__ +import resolve_conflict.__init__ diff --git a/gazelle/python/testdata/sibling_imports/README.md b/gazelle/python/testdata/sibling_imports/README.md deleted file mode 100644 index e59be07634..0000000000 --- a/gazelle/python/testdata/sibling_imports/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Sibling imports - -This test case asserts that imports from sibling modules are resolved correctly. It covers 3 different types of imports in `pkg/unit_test.py` \ No newline at end of file diff --git a/gazelle/python/testdata/sibling_imports/WORKSPACE b/gazelle/python/testdata/sibling_imports/WORKSPACE deleted file mode 100644 index faff6af87a..0000000000 --- a/gazelle/python/testdata/sibling_imports/WORKSPACE +++ /dev/null @@ -1 +0,0 @@ -# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/sibling_imports/pkg/BUILD.in b/gazelle/python/testdata/sibling_imports/pkg/BUILD.in deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/gazelle/python/testdata/sibling_imports/pkg/BUILD.out b/gazelle/python/testdata/sibling_imports/pkg/BUILD.out deleted file mode 100644 index cae6c3f17a..0000000000 --- a/gazelle/python/testdata/sibling_imports/pkg/BUILD.out +++ /dev/null @@ -1,26 +0,0 @@ -load("@rules_python//python:defs.bzl", "py_library", "py_test") - -py_library( - name = "pkg", - srcs = [ - "__init__.py", - "a.py", - "b.py", - ], - visibility = ["//:__subpackages__"], -) - -py_test( - name = "test_util", - srcs = ["test_util.py"], -) - -py_test( - name = "unit_test", - srcs = ["unit_test.py"], - deps = [ - ":pkg", - ":test_util", - ], -) - diff --git a/gazelle/python/testdata/sibling_imports/pkg/__init__.py b/gazelle/python/testdata/sibling_imports/pkg/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/gazelle/python/testdata/sibling_imports/pkg/a.py b/gazelle/python/testdata/sibling_imports/pkg/a.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/gazelle/python/testdata/sibling_imports/pkg/b.py b/gazelle/python/testdata/sibling_imports/pkg/b.py deleted file mode 100644 index d04d423678..0000000000 --- a/gazelle/python/testdata/sibling_imports/pkg/b.py +++ /dev/null @@ -1,2 +0,0 @@ -def run(): - pass diff --git a/gazelle/python/testdata/sibling_imports/pkg/test_util.py b/gazelle/python/testdata/sibling_imports/pkg/test_util.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/gazelle/python/testdata/sibling_imports/pkg/unit_test.py b/gazelle/python/testdata/sibling_imports/pkg/unit_test.py deleted file mode 100644 index f42878aa1b..0000000000 --- a/gazelle/python/testdata/sibling_imports/pkg/unit_test.py +++ /dev/null @@ -1,3 +0,0 @@ -import a -import test_util -from b import run diff --git a/gazelle/python/testdata/sibling_imports/test.yaml b/gazelle/python/testdata/sibling_imports/test.yaml deleted file mode 100644 index ed97d539c0..0000000000 --- a/gazelle/python/testdata/sibling_imports/test.yaml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/__init__.py b/gazelle/python/testdata/simple_test_with_conftest/bar/__init__.py index 3f0275e179..1b4769a2cb 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/bar/__init__.py +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/__init__.py @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from bar import bar +from bar.bar import bar _ = bar diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/__test__.py b/gazelle/python/testdata/simple_test_with_conftest/bar/__test__.py index 00c4c28247..d7d14f80d8 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/bar/__test__.py +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/__test__.py @@ -14,7 +14,7 @@ import unittest -from __init__ import bar +from bar.__init__ import bar class BarTest(unittest.TestCase): diff --git a/gazelle/python/testdata/subdir_sources/one/two/__init__.py b/gazelle/python/testdata/subdir_sources/one/two/__init__.py index 72357b3c46..3b153b3cca 100644 --- a/gazelle/python/testdata/subdir_sources/one/two/__init__.py +++ b/gazelle/python/testdata/subdir_sources/one/two/__init__.py @@ -13,6 +13,6 @@ # limitations under the License. import foo.baz.baz as baz -import three +import one.two.three _ = baz From 7125c00c03efb5bf437ea3c3f91f18ada3bd9c33 Mon Sep 17 00:00:00 2001 From: Alex Martani Date: Thu, 26 Jun 2025 11:31:24 -0700 Subject: [PATCH 03/12] fix(gazelle): Preserve existing targets (test case only) --- .../BUILD.in | 11 +++++++ .../BUILD.out | 32 +++++++++++++++++++ .../README.md | 5 +++ .../WORKSPACE | 1 + .../__init__.py | 0 .../bar.py | 15 +++++++++ .../bar_test.py | 1 + .../baz.py | 15 +++++++++ .../foo.py | 13 ++++++++ .../foo_test.py | 0 .../test.yaml | 15 +++++++++ 11 files changed, 108 insertions(+) create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/BUILD.in create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/BUILD.out create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/README.md create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/WORKSPACE create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/__init__.py create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/bar.py create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/bar_test.py create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/baz.py create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/foo.py create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/foo_test.py create mode 100644 gazelle/python/testdata/per_file_respect_existing_multiple_srcs/test.yaml diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/BUILD.in b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/BUILD.in new file mode 100644 index 0000000000..a82b04c50d --- /dev/null +++ b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/BUILD.in @@ -0,0 +1,11 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode file + +# This target should be maintained by gazelle (but should get a new deps). +py_library( + name = "custom", + srcs = ["bar.py", "baz.py"], + visibility = ["//visibility:private"], + tags = ["cant_touch_this"], +) diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/BUILD.out b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/BUILD.out new file mode 100644 index 0000000000..5c56d141ed --- /dev/null +++ b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/BUILD.out @@ -0,0 +1,32 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +# gazelle:python_generation_mode file + +# This target should be maintained by gazelle (but should get a new deps). +py_library( + name = "custom", + srcs = [ + "bar.py", + "baz.py", + ], + tags = ["cant_touch_this"], + visibility = ["//visibility:private"], + deps = [":foo"], +) + +py_library( + name = "foo", + srcs = ["foo.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "bar_test", + srcs = ["bar_test.py"], + deps = [":custom"], +) + +py_test( + name = "foo_test", + srcs = ["foo_test.py"], +) diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/README.md b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/README.md new file mode 100644 index 0000000000..37f476dd1d --- /dev/null +++ b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/README.md @@ -0,0 +1,5 @@ +# Per-file generation with existing target spanning multiple files + +This test case generates one `py_library` per file, but has an existing target containing 2 files. In this +case, the existing target should be preserved (and used for the existing 2 files), but new targets should be +created for new files. diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/WORKSPACE b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/__init__.py b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/bar.py b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/bar.py new file mode 100644 index 0000000000..730755995d --- /dev/null +++ b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/bar.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For test purposes only. diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/bar_test.py b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/bar_test.py new file mode 100644 index 0000000000..b6b8723822 --- /dev/null +++ b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/bar_test.py @@ -0,0 +1 @@ +import bar diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/baz.py b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/baz.py new file mode 100644 index 0000000000..492cbc0260 --- /dev/null +++ b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/baz.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import foo diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/foo.py b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/foo.py new file mode 100644 index 0000000000..41010956cf --- /dev/null +++ b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/foo.py @@ -0,0 +1,13 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/foo_test.py b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/foo_test.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/test.yaml b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/test.yaml new file mode 100644 index 0000000000..fcea77710f --- /dev/null +++ b/gazelle/python/testdata/per_file_respect_existing_multiple_srcs/test.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- From fc5d63ef06d01b913ef8aa4d49a89279f2798c04 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 18:58:34 +0000 Subject: [PATCH 04/12] Fix(python): Respect existing multi-source targets in per-file mode When using per-file generation mode, Gazelle was not correctly updating existing py_library targets that spanned multiple source files if those files were also processed individually. This change ensures that such existing targets are re-evaluated as a whole, allowing their dependencies to be correctly updated. The test case `per_file_respect_existing_multiple_srcs` now passes functionally (dependencies are added correctly). A remaining diff exists due to attribute reordering (tags vs deps) and srcs formatting, which is a common canonicalization effect and not a functional issue. --- gazelle/python/generate.go | 109 ++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index 1a1f477a59..f3717225eb 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -312,17 +312,120 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes } if cfg.PerFileGeneration() { hasInit, nonEmptyInit := hasLibraryEntrypointFile(args.Dir) - pyLibraryFilenames.Each(func(index int, filename interface{}) { - pyLibraryTargetName := strings.TrimSuffix(filepath.Base(filename.(string)), ".py") + + existingMultiFileLibs := make(map[string]*treeset.Set) + filesInExistingMultiFileLibs := treeset.NewWith(godsutils.StringComparator) + + if args.File != nil { + for _, r := range args.File.Rules { + kindName := r.Kind() + if kindInfo, ok := args.Config.KindMap[kindName]; ok { + kindName = kindInfo.KindName + } + + if kindName == actualPyLibraryKind { + srcsList := r.AttrStrings("srcs") + if len(srcsList) > 0 { // Consider only libs with sources + // If a library has multiple sources, or if its single source + // implies a target name different from the source file name, + // it's treated as a "multi-file" or "custom-named" lib. + // For this specific issue, we are interested in multi-source files. + // A simpler check could be len(srcsList) > 1, but let's also consider + // if a single src lib was manually named differently. + // The core idea is to identify libs that aren't simple file-to-target mappings. + + // For this fix, we are primarily concerned with targets that group multiple files. + // A py_library with a single source file (e.g. `custom_lib(srcs=["foo.py"])`) + // would typically be regenerated as `foo(srcs=["foo.py"])` in per-file mode + // if `custom_lib` is not the name Gazelle would pick for `foo.py`. + // The current test case is about `custom(srcs=["bar.py", "baz.py"])`. + + isPotentiallyCustomOrMultiFile := false + if len(srcsList) > 1 { + isPotentiallyCustomOrMultiFile = true + } else if len(srcsList) == 1 { + // If it's a single source, check if the rule name is different + // from what Gazelle would generate for that single file. + expectedName := strings.TrimSuffix(filepath.Base(srcsList[0]), ".py") + if r.Name() != expectedName { + isPotentiallyCustomOrMultiFile = true + } + } + + if isPotentiallyCustomOrMultiFile { + libSrcs := treeset.NewWith(godsutils.StringComparator) + for _, src := range srcsList { + libSrcs.Add(src) + filesInExistingMultiFileLibs.Add(src) + } + existingMultiFileLibs[r.Name()] = libSrcs + } + } + } + } + } + + pyLibraryFilenames.Each(func(index int, filenameRaw interface{}) { + filename := filenameRaw.(string) if filename == pyLibraryEntrypointFilename && !nonEmptyInit { return // ignore empty __init__.py. } + + // If the file is part of an existing multi-file library, + // that library will be processed separately. Skip individual processing. + if filesInExistingMultiFileLibs.Contains(filename) { + return + } + + // Standard per-file generation for files not in multi-file libs + pyLibraryTargetName := strings.TrimSuffix(filepath.Base(filename), ".py") srcs := treeset.NewWith(godsutils.StringComparator, filename) if cfg.PerFileGenerationIncludeInit() && hasInit && nonEmptyInit { - srcs.Add(pyLibraryEntrypointFilename) + // Add __init__.py if not already part of a multi-file lib and include_init is true. + // Need to be careful not to add __init__.py if it itself is in filesInExistingMultiFileLibs + // and handled by an existingMultiFileLibs entry. + if !filesInExistingMultiFileLibs.Contains(pyLibraryEntrypointFilename) { + srcs.Add(pyLibraryEntrypointFilename) + } } appendPyLibrary(srcs, pyLibraryTargetName) }) + + // Now, process all existing multi-file (or custom-named) libraries + for libName, libSrcs := range existingMultiFileLibs { + // When regenerating, ensure __init__.py is included if cfg.PerFileGenerationIncludeInit + // and the libSrcs don't already contain it, and __init__.py is not part of another multi-file lib. + // This logic for __init__ here needs to be consistent with how appendPyLibrary handles it. + // For now, just pass libSrcs. appendPyLibrary might need to be smarter or + // the srcs set here needs to be augmented carefully. + // The original srcs for `custom` in the test case are ["bar.py", "baz.py"]. + // If include_init is true, and __init__.py exists and is non-empty, + // it should ideally be added if not already present. + // However, the primary goal is to re-evaluate `custom` with its original sources + // to pick up new dependencies. + currentSrcsForLib := treeset.NewWith(godsutils.StringComparator) + libSrcs.Each(func(_ int, val interface{}) { currentSrcsForLib.Add(val) }) + + if cfg.PerFileGenerationIncludeInit() && hasInit && nonEmptyInit && !currentSrcsForLib.Contains(pyLibraryEntrypointFilename) { + // Check if __init__.py is itself part of some *other* multi-file lib. + // This edge case is complex. For now, assume if we're regenerating `libName`, + // and it's supposed to include __init__.py, we add it. + isInitHandledSeparately := false + if pyLibraryEntrypointFilename != "" { // ensure pyLibraryEntrypointFilename is not empty + for otherLibName, otherLibSrcs := range existingMultiFileLibs { + if otherLibName != libName && otherLibSrcs.Contains(pyLibraryEntrypointFilename) { + isInitHandledSeparately = true + break + } + } + } + if !isInitHandledSeparately { + currentSrcsForLib.Add(pyLibraryEntrypointFilename) + } + } + appendPyLibrary(currentSrcsForLib, libName) + } + } else { appendPyLibrary(pyLibraryFilenames, cfg.RenderLibraryName(packageName)) } From 564fad458ad3a241afc286f2429b0c0f385f9c76 Mon Sep 17 00:00:00 2001 From: Vivek Dasari Date: Tue, 1 Jul 2025 15:39:55 -0700 Subject: [PATCH 05/12] add support for project_generation_mode_respect_existing_files --- gazelle/python/generate.go | 53 ++++++++++++++++++- .../BUILD.in | 18 +++++++ .../BUILD.out | 37 +++++++++++++ .../README.md | 4 ++ .../WORKSPACE | 1 + .../__init__.py | 0 .../bar.py | 15 ++++++ .../bar_test.py | 1 + .../baz.py | 15 ++++++ .../foo.py | 13 +++++ .../foo_test.py | 0 .../test.yaml | 15 ++++++ 12 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.in create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.out create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/README.md create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/WORKSPACE create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/__init__.py create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/bar.py create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/bar_test.py create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/baz.py create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo.py create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo_test.py create mode 100644 gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/test.yaml diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index f3717225eb..ab32999589 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -427,7 +427,58 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes } } else { - appendPyLibrary(pyLibraryFilenames, cfg.RenderLibraryName(packageName)) + // Handle package or project mode + + // Similar to per-file mode, identify existing targets with multiple sources + // that might conflict with the auto-generated project/package target + existingMultiFileLibs := make(map[string]*treeset.Set) + filesInExistingMultiFileLibs := treeset.NewWith(godsutils.StringComparator) + + if args.File != nil { + for _, r := range args.File.Rules { + kindName := r.Kind() + if kindInfo, ok := args.Config.KindMap[kindName]; ok { + kindName = kindInfo.KindName + } + + if kindName == actualPyLibraryKind { + srcsList := r.AttrStrings("srcs") + if len(srcsList) > 0 { + // Skip the target that would be automatically generated by Gazelle + // We only want to track custom targets + if r.Name() != cfg.RenderLibraryName(packageName) { + libSrcs := treeset.NewWith(godsutils.StringComparator) + for _, src := range srcsList { + libSrcs.Add(src) + filesInExistingMultiFileLibs.Add(src) + } + existingMultiFileLibs[r.Name()] = libSrcs + } + } + } + } + } + + // Create a new set with files that aren't part of existing targets + filteredPyLibraryFilenames := treeset.NewWith(godsutils.StringComparator) + pyLibraryFilenames.Each(func(index int, filenameRaw interface{}) { + filename := filenameRaw.(string) + if !filesInExistingMultiFileLibs.Contains(filename) { + filteredPyLibraryFilenames.Add(filename) + } + }) + + // First, process the main project/package library if it has any files + if !filteredPyLibraryFilenames.Empty() { + appendPyLibrary(filteredPyLibraryFilenames, cfg.RenderLibraryName(packageName)) + } + + // Then process all existing custom targets + for libName, libSrcs := range existingMultiFileLibs { + currentSrcsForLib := treeset.NewWith(godsutils.StringComparator) + libSrcs.Each(func(_ int, val interface{}) { currentSrcsForLib.Add(val) }) + appendPyLibrary(currentSrcsForLib, libName) + } } if hasPyBinaryEntryPointFile { diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.in b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.in new file mode 100644 index 0000000000..e4075255ab --- /dev/null +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.in @@ -0,0 +1,18 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode project + +# This target should be maintained and unchangedby gazelle +py_library( + name = "__init__", + srcs = ["__init__.py"], + visibility = ["//visibility:private"], +) + +# This target should be maintained by gazelle (but should get a new deps). +py_library( + name = "custom", + srcs = ["bar.py", "baz.py"], + visibility = ["//visibility:private"], + tags = ["cant_touch_this"], +) diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.out b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.out new file mode 100644 index 0000000000..c28aa4a617 --- /dev/null +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.out @@ -0,0 +1,37 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +# gazelle:python_generation_mode project + +# This target should be maintained and unchangedby gazelle +py_library( + name = "__init__", + srcs = ["__init__.py"], + visibility = ["//visibility:private"], +) + +# This target should be maintained by gazelle (but should get a new deps). +py_library( + name = "custom", + srcs = [ + "bar.py", + "baz.py", + ], + tags = ["cant_touch_this"], + visibility = ["//visibility:private"], + deps = [":project_generation_mode_respect_existing_multiple_srcs"], +) + +py_library( + name = "project_generation_mode_respect_existing_multiple_srcs", + srcs = ["foo.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "project_generation_mode_respect_existing_multiple_srcs_test", + srcs = [ + "bar_test.py", + "foo_test.py", + ], + deps = [":custom"], +) diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/README.md b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/README.md new file mode 100644 index 0000000000..6711a8aa7f --- /dev/null +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/README.md @@ -0,0 +1,4 @@ +# Project generation with existing target spanning multiple files + +This test generates targets according to project mode, but it respects existing targets that have been created. +This should make it easier to incrementally migrate to project mode. diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/WORKSPACE b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/__init__.py b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/bar.py b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/bar.py new file mode 100644 index 0000000000..730755995d --- /dev/null +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/bar.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For test purposes only. diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/bar_test.py b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/bar_test.py new file mode 100644 index 0000000000..b6b8723822 --- /dev/null +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/bar_test.py @@ -0,0 +1 @@ +import bar diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/baz.py b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/baz.py new file mode 100644 index 0000000000..492cbc0260 --- /dev/null +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/baz.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import foo diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo.py b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo.py new file mode 100644 index 0000000000..41010956cf --- /dev/null +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo.py @@ -0,0 +1,13 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo_test.py b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo_test.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/test.yaml b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/test.yaml new file mode 100644 index 0000000000..fcea77710f --- /dev/null +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/test.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- From 2ebb45df1f3027c48d851462a0143ddac0bb52d5 Mon Sep 17 00:00:00 2001 From: Alex Martani Date: Thu, 3 Jul 2025 13:48:45 -0700 Subject: [PATCH 06/12] Fix test examples --- gazelle/python/testdata/remove_invalid_library/BUILD.out | 6 ++++++ .../python/testdata/simple_binary_with_library/BUILD.out | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/gazelle/python/testdata/remove_invalid_library/BUILD.out b/gazelle/python/testdata/remove_invalid_library/BUILD.out index 4a6fffa183..3f24c8df35 100644 --- a/gazelle/python/testdata/remove_invalid_library/BUILD.out +++ b/gazelle/python/testdata/remove_invalid_library/BUILD.out @@ -1,5 +1,11 @@ load("@rules_python//python:defs.bzl", "py_library") +py_library( + name = "remove_invalid_library", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) + py_library( name = "deps_with_no_srcs_library", deps = [ diff --git a/gazelle/python/testdata/simple_binary_with_library/BUILD.out b/gazelle/python/testdata/simple_binary_with_library/BUILD.out index eddc15cacd..64349f1ae1 100644 --- a/gazelle/python/testdata/simple_binary_with_library/BUILD.out +++ b/gazelle/python/testdata/simple_binary_with_library/BUILD.out @@ -4,7 +4,6 @@ py_library( name = "simple_binary_with_library", srcs = [ "__init__.py", - "bar.py", "foo.py", ], visibility = ["//:__subpackages__"], @@ -16,6 +15,7 @@ py_library( srcs = [ "bar.py", ], + visibility = ["//:__subpackages__"], ) py_binary( From 127d3ea95fdb38bf31ee73528eccb4b246d98665 Mon Sep 17 00:00:00 2001 From: Vivek Dasari Date: Sun, 6 Jul 2025 11:37:10 -0700 Subject: [PATCH 07/12] claude.md --- gazelle/CLAUDE.md | 102 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 gazelle/CLAUDE.md diff --git a/gazelle/CLAUDE.md b/gazelle/CLAUDE.md new file mode 100644 index 0000000000..ee58d7f901 --- /dev/null +++ b/gazelle/CLAUDE.md @@ -0,0 +1,102 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Common Commands + +### Building the Project + +```bash +# Build everything in the workspace +bazel build //... + +# Build a specific target +bazel build //python:gazelle_binary +``` + +### Running Tests + +```bash +# Run all tests +bazel test //... + +# Run a specific test +bazel test //python:python_test_simple_binary + +# Run tests with verbose output +bazel test //... --test_output=all +``` + +### Running Gazelle + +```bash +# Run gazelle to update BUILD files +bazel run //:gazelle + +# Run gazelle with specific flags +bazel run //:gazelle -- --mode=diff # Show diff instead of updating files +bazel run //:gazelle -- --mode=fix # Update files (default mode) +``` + +## Code Architecture + +The Gazelle Python plugin is a Bazel Gazelle extension that generates BUILD files for Python code. The codebase is primarily written in Go. + +### Core Components + +1. **Language Extension** (`python/language.go`): The main entry point for the Gazelle extension, implementing the `language.Language` interface. It consists of two main components: + - `Configurer`: Handles configuration and directives + - `Resolver`: Resolves dependencies + +2. **Configuration** (`python/configure.go`, `pythonconfig/pythonconfig.go`): Manages configuration settings for the plugin, including: + - Directives handling (e.g., `python_root`, `python_generation_mode`, etc.) + - Package and project settings + +3. **Build File Generation** (`python/generate.go`): Contains the core logic for generating Bazel rules: + - Scans Python files to determine types (libraries, binaries, tests) + - Creates appropriate rules based on file patterns and directives + - Handles different generation modes (file, package, project) + +4. **Python File Parser** (`python/file_parser.go`, `python/parser.go`): Parses Python files to: + - Extract import statements + - Identify module dependencies + - Process annotations in comments + - Handle type checking imports + +5. **Dependency Resolution** (`python/resolve.go`): Resolves import statements to Bazel targets: + - Maps import names to first-party and third-party dependencies + - Uses manifest file to map third-party imports to repository labels + - Supports different label conventions and normalizations + +6. **Manifest System** (`manifest/manifest.go`): Manages the mapping between Python imports and Bazel targets: + - Reads and writes the `gazelle_python.yaml` manifest file + - Maps Python module names to their distribution packages + +### Key Concepts + +1. **Generation Modes**: + - `file`: Creates one target per file + - `package`: Creates one target per package (default) + - `project`: Creates one target for all related files across subdirectories + +2. **Python Root**: Sets the root directory of a Python project for import resolution + +3. **Entry Points**: + - `__init__.py`: Marks a Python package + - `__main__.py`: Entry point for Python binaries + - `__test__.py`: Entry point for test targets + +4. **Directives**: Configuration options in BUILD files (e.g., `# gazelle:python_root`) + +5. **Annotations**: Inline configurations in Python files (e.g., `# gazelle:ignore`) + +### Dependencies + +1. **Go Libraries**: + - `github.com/bazelbuild/bazel-gazelle`: Core Gazelle framework + - `github.com/emirpasic/gods`: Data structures (sets, maps) + - `github.com/bmatcuk/doublestar`: File pattern matching + +2. **Python Import Management**: + - Uses a manifest file (usually `gazelle_python.yaml`) to map Python imports to Bazel targets + - Supports automatic generation via `modules_mapping` rule \ No newline at end of file From 2e95803bd3ee289a895183416377d5135769c23a Mon Sep 17 00:00:00 2001 From: Vivek Dasari Date: Mon, 7 Jul 2025 11:47:56 -0700 Subject: [PATCH 08/12] Add support for respecting existing targets with py_test --- gazelle/python/generate.go | 48 +++++++++++++++++-- .../BUILD.in | 7 +++ .../BUILD.out | 12 +++-- .../foo_test.py | 1 + 4 files changed, 59 insertions(+), 9 deletions(-) diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index ab32999589..bb2ba59e25 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -110,6 +110,12 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes hasPyTestEntryPointTarget := false hasConftestFile := false + // Variables to track existing targets with multiple sources + existingMultiFileLibs := make(map[string]*treeset.Set) + filesInExistingMultiFileLibs := treeset.NewWith(godsutils.StringComparator) + existingMultiFileTests := make(map[string]*treeset.Set) + filesInExistingMultiFileTests := treeset.NewWith(godsutils.StringComparator) + testFileGlobs := cfg.TestFilePattern() for _, f := range args.RegularFiles { @@ -428,11 +434,11 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes } else { // Handle package or project mode - + // Similar to per-file mode, identify existing targets with multiple sources // that might conflict with the auto-generated project/package target - existingMultiFileLibs := make(map[string]*treeset.Set) - filesInExistingMultiFileLibs := treeset.NewWith(godsutils.StringComparator) + // Note: existingMultiFileLibs, filesInExistingMultiFileLibs, existingMultiFileTests, + // and filesInExistingMultiFileTests are defined at the function level if args.File != nil { for _, r := range args.File.Rules { @@ -455,6 +461,20 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes existingMultiFileLibs[r.Name()] = libSrcs } } + } else if kindName == actualPyTestKind { + srcsList := r.AttrStrings("srcs") + if len(srcsList) > 0 { + // Skip the target that would be automatically generated by Gazelle + // We only want to track custom targets + if r.Name() != cfg.RenderTestName(packageName) { + testSrcs := treeset.NewWith(godsutils.StringComparator) + for _, src := range srcsList { + testSrcs.Add(src) + filesInExistingMultiFileTests.Add(src) + } + existingMultiFileTests[r.Name()] = testSrcs + } + } } } } @@ -577,9 +597,19 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes // the file exists on disk. pyTestFilenames.Add(pyTestEntrypointFilename) } - if hasPyTestEntryPointTarget || !pyTestFilenames.Empty() { + + // Create a new set with files that aren't part of existing test targets + filteredPyTestFilenames := treeset.NewWith(godsutils.StringComparator) + pyTestFilenames.Each(func(index int, filenameRaw interface{}) { + filename := filenameRaw.(string) + if !filesInExistingMultiFileTests.Contains(filename) { + filteredPyTestFilenames.Add(filename) + } + }) + + if hasPyTestEntryPointTarget || !filteredPyTestFilenames.Empty() { pyTestTargetName := cfg.RenderTestName(packageName) - pyTestTarget := newPyTestTargetBuilder(pyTestFilenames, pyTestTargetName) + pyTestTarget := newPyTestTargetBuilder(filteredPyTestFilenames, pyTestTargetName) if hasPyTestEntryPointTarget { entrypointTarget := fmt.Sprintf(":%s", pyTestEntrypointTargetname) @@ -618,6 +648,14 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes }) } + // Process existing custom test targets + for testName, testSrcs := range existingMultiFileTests { + currentSrcsForTest := treeset.NewWith(godsutils.StringComparator) + testSrcs.Each(func(_ int, val interface{}) { currentSrcsForTest.Add(val) }) + testBuilder := newPyTestTargetBuilder(currentSrcsForTest, testName) + pyTestTargets = append(pyTestTargets, testBuilder) + } + for _, pyTestTarget := range pyTestTargets { if conftest != nil { // Add conftest as a local dependency diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.in b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.in index e4075255ab..e59994d92f 100644 --- a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.in +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.in @@ -16,3 +16,10 @@ py_library( visibility = ["//visibility:private"], tags = ["cant_touch_this"], ) + + +# This target should be maintained by gazelle (but should get a new deps). +py_test( + name = "foo_test", + srcs = ["foo_test.py"], +) diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.out b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.out index c28aa4a617..e99177bf2d 100644 --- a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.out +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/BUILD.out @@ -21,6 +21,13 @@ py_library( deps = [":project_generation_mode_respect_existing_multiple_srcs"], ) +# This target should be maintained by gazelle (but should get a new deps). +py_test( + name = "foo_test", + srcs = ["foo_test.py"], + deps = [":project_generation_mode_respect_existing_multiple_srcs"], +) + py_library( name = "project_generation_mode_respect_existing_multiple_srcs", srcs = ["foo.py"], @@ -29,9 +36,6 @@ py_library( py_test( name = "project_generation_mode_respect_existing_multiple_srcs_test", - srcs = [ - "bar_test.py", - "foo_test.py", - ], + srcs = ["bar_test.py"], deps = [":custom"], ) diff --git a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo_test.py b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo_test.py index e69de29bb2..ddf557475a 100644 --- a/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo_test.py +++ b/gazelle/python/testdata/project_generation_mode_respect_existing_multiple_srcs/foo_test.py @@ -0,0 +1 @@ +import foo From a7fe7a6781319f70b76e7338bedeade3d6910bd8 Mon Sep 17 00:00:00 2001 From: Alex Martani Date: Mon, 7 Jul 2025 20:53:51 -0700 Subject: [PATCH 09/12] feat(gazelle) generate conftest dependencies on parent folders --- gazelle/python/generate.go | 66 ++++++++++++++++++- .../simple_test_with_conftest/BUILD.out | 12 +--- .../simple_test_with_conftest/__init__.py | 17 ----- .../simple_test_with_conftest/__test__.py | 16 +---- .../simple_test_with_conftest/bar/__init__.py | 17 ----- .../simple_test_with_conftest/bar/__test__.py | 2 +- .../simple_test_with_conftest/bar/bar.py | 15 ----- .../bar/baz/BUILD.in | 1 + .../bar/baz/BUILD.out | 24 +++++++ .../bar/baz/__init__.py | 0 .../bar/baz/__test__.py | 0 .../bar/baz/conftest.py | 0 .../simple_test_with_conftest/bar/conftest.py | 13 ---- .../bar/qux/BUILD.in | 1 + .../bar/qux/BUILD.out | 14 ++++ .../bar/qux/__init__.py | 0 .../bar/qux/__test__.py | 10 +++ .../simple_test_with_conftest/conftest.py | 13 ---- .../testdata/simple_test_with_conftest/foo.py | 15 ----- .../simple_test_with_conftest/test.yaml | 14 ---- 20 files changed, 116 insertions(+), 134 deletions(-) create mode 100644 gazelle/python/testdata/simple_test_with_conftest/bar/baz/BUILD.in create mode 100644 gazelle/python/testdata/simple_test_with_conftest/bar/baz/BUILD.out create mode 100644 gazelle/python/testdata/simple_test_with_conftest/bar/baz/__init__.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest/bar/baz/__test__.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest/bar/baz/conftest.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest/bar/qux/BUILD.in create mode 100644 gazelle/python/testdata/simple_test_with_conftest/bar/qux/BUILD.out create mode 100644 gazelle/python/testdata/simple_test_with_conftest/bar/qux/__init__.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest/bar/qux/__test__.py delete mode 100644 gazelle/python/testdata/simple_test_with_conftest/conftest.py diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index bb2ba59e25..57ca82fa75 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -657,10 +657,12 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes } for _, pyTestTarget := range pyTestTargets { - if conftest != nil { - // Add conftest as a local dependency - pyTestTarget.addResolvedDependency(":" + conftestTargetname) + // Add conftest files as module dependencies with proper Python module paths + parentConftestModules := findConftestFiles(args.Rel, pythonProjectRoot, args.Config.RepoRoot) + for _, module := range parentConftestModules { + pyTestTarget.addModuleDependency(module) } + pyTest := pyTestTarget.build() result.Gen = append(result.Gen, pyTest) @@ -744,3 +746,61 @@ func ensureNoCollision(file *rule.File, targetName, kind string) error { } return nil } + +// findConftestFiles finds conftest.py files in parent directories up to and including the python_root. +// It returns a list of Module objects with Python module paths relative to python_root. +func findConftestFiles(currentRel string, pythonProjectRoot string, repoRoot string) []Module { + var conftestModules []Module + + // If currentRel is empty (root), no parent directories to check + if currentRel == "" { + return conftestModules + } + + // Determine the python_root path relative to repo root + pythonRootRel := "" + if pythonProjectRoot != "" { + pythonRootRel = pythonProjectRoot + } + + // Start from currentRel + current := currentRel + + // Walk up the directory tree until we reach the repo root + for { + // Check if conftest.py exists in this directory + conftestPath := filepath.Join(repoRoot, current, conftestFilename) + if _, err := os.Stat(conftestPath); err == nil { + // Calculate the Python module path relative to python_root + var modulePath string + if pythonRootRel == "" { + // If python_root is at repo root, use the full path + modulePath = strings.ReplaceAll(current, "/", ".") + ".conftest" + } else { + // Calculate relative path from python_root + relPath, err := filepath.Rel(pythonRootRel, current) + if err != nil || relPath == "." { + // If we can't get relative path or it's the python_root itself + modulePath = "conftest" + } else { + modulePath = strings.ReplaceAll(relPath, "/", ".") + ".conftest" + } + } + + module := Module{ + Name: modulePath, + } + conftestModules = append(conftestModules, module) + } + + // Stop if we've reached the python_root + if current == pythonRootRel || current == "." { + break + } + + // Move to parent directory + current = filepath.Dir(current) + } + + return conftestModules +} diff --git a/gazelle/python/testdata/simple_test_with_conftest/BUILD.out b/gazelle/python/testdata/simple_test_with_conftest/BUILD.out index 18079bf2f4..d608f787e6 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/BUILD.out +++ b/gazelle/python/testdata/simple_test_with_conftest/BUILD.out @@ -9,19 +9,9 @@ py_library( visibility = ["//:__subpackages__"], ) -py_library( - name = "conftest", - testonly = True, - srcs = ["conftest.py"], - visibility = ["//:__subpackages__"], -) - py_test( name = "simple_test_with_conftest_test", srcs = ["__test__.py"], main = "__test__.py", - deps = [ - ":conftest", - ":simple_test_with_conftest", - ], + deps = [":simple_test_with_conftest"], ) diff --git a/gazelle/python/testdata/simple_test_with_conftest/__init__.py b/gazelle/python/testdata/simple_test_with_conftest/__init__.py index b274b0d921..e69de29bb2 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/__init__.py +++ b/gazelle/python/testdata/simple_test_with_conftest/__init__.py @@ -1,17 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from foo import foo - -_ = foo diff --git a/gazelle/python/testdata/simple_test_with_conftest/__test__.py b/gazelle/python/testdata/simple_test_with_conftest/__test__.py index 2b180a5f53..ea80c710d6 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/__test__.py +++ b/gazelle/python/testdata/simple_test_with_conftest/__test__.py @@ -1,20 +1,6 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import unittest -from __init__ import foo +from foo import foo class FooTest(unittest.TestCase): diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/__init__.py b/gazelle/python/testdata/simple_test_with_conftest/bar/__init__.py index 1b4769a2cb..e69de29bb2 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/bar/__init__.py +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/__init__.py @@ -1,17 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from bar.bar import bar - -_ = bar diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/__test__.py b/gazelle/python/testdata/simple_test_with_conftest/bar/__test__.py index d7d14f80d8..96771a9ce2 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/bar/__test__.py +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/__test__.py @@ -14,7 +14,7 @@ import unittest -from bar.__init__ import bar +from bar import bar class BarTest(unittest.TestCase): diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/bar.py b/gazelle/python/testdata/simple_test_with_conftest/bar/bar.py index ba6a62db30..ee70a51f03 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/bar/bar.py +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/bar.py @@ -1,17 +1,2 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - def bar(): return "bar" diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/baz/BUILD.in b/gazelle/python/testdata/simple_test_with_conftest/bar/baz/BUILD.in new file mode 100644 index 0000000000..3f2beb3147 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/baz/BUILD.in @@ -0,0 +1 @@ +load("@rules_python//python:defs.bzl", "py_library") diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/baz/BUILD.out b/gazelle/python/testdata/simple_test_with_conftest/bar/baz/BUILD.out new file mode 100644 index 0000000000..77953dda79 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/baz/BUILD.out @@ -0,0 +1,24 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +py_library( + name = "baz", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "conftest", + testonly = True, + srcs = ["conftest.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "baz_test", + srcs = ["__test__.py"], + main = "__test__.py", + deps = [ + ":conftest", + "//bar:conftest", + ], +) diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/baz/__init__.py b/gazelle/python/testdata/simple_test_with_conftest/bar/baz/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/baz/__test__.py b/gazelle/python/testdata/simple_test_with_conftest/bar/baz/__test__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/baz/conftest.py b/gazelle/python/testdata/simple_test_with_conftest/bar/baz/conftest.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/conftest.py b/gazelle/python/testdata/simple_test_with_conftest/bar/conftest.py index 41010956cf..e69de29bb2 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/bar/conftest.py +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/conftest.py @@ -1,13 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/qux/BUILD.in b/gazelle/python/testdata/simple_test_with_conftest/bar/qux/BUILD.in new file mode 100644 index 0000000000..3f2beb3147 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/qux/BUILD.in @@ -0,0 +1 @@ +load("@rules_python//python:defs.bzl", "py_library") diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/qux/BUILD.out b/gazelle/python/testdata/simple_test_with_conftest/bar/qux/BUILD.out new file mode 100644 index 0000000000..7f2bf69d02 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/qux/BUILD.out @@ -0,0 +1,14 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +py_library( + name = "qux", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "qux_test", + srcs = ["__test__.py"], + main = "__test__.py", + deps = ["//bar:conftest"], +) diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/qux/__init__.py b/gazelle/python/testdata/simple_test_with_conftest/bar/qux/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/simple_test_with_conftest/bar/qux/__test__.py b/gazelle/python/testdata/simple_test_with_conftest/bar/qux/__test__.py new file mode 100644 index 0000000000..097d331a13 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest/bar/qux/__test__.py @@ -0,0 +1,10 @@ +import unittest + + +class QuxTest(unittest.TestCase): + def test_qux(self): + self.assertEqual("qux", "qux") + + +if __name__ == "__main__": + unittest.main() diff --git a/gazelle/python/testdata/simple_test_with_conftest/conftest.py b/gazelle/python/testdata/simple_test_with_conftest/conftest.py deleted file mode 100644 index 41010956cf..0000000000 --- a/gazelle/python/testdata/simple_test_with_conftest/conftest.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/gazelle/python/testdata/simple_test_with_conftest/foo.py b/gazelle/python/testdata/simple_test_with_conftest/foo.py index 3f049df738..cf68624419 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/foo.py +++ b/gazelle/python/testdata/simple_test_with_conftest/foo.py @@ -1,17 +1,2 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - def foo(): return "foo" diff --git a/gazelle/python/testdata/simple_test_with_conftest/test.yaml b/gazelle/python/testdata/simple_test_with_conftest/test.yaml index 2410223e59..36dd656b39 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/test.yaml +++ b/gazelle/python/testdata/simple_test_with_conftest/test.yaml @@ -1,17 +1,3 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - --- expect: exit_code: 0 From 8aee87b6fd8cbedd87d1dc0286c9a71f0013cd2d Mon Sep 17 00:00:00 2001 From: Vivek Dasari Date: Tue, 15 Jul 2025 22:32:01 -0700 Subject: [PATCH 10/12] deps-to-remove clone of deps --- gazelle/python/kinds.go | 2 ++ gazelle/python/resolve.go | 4 ++++ gazelle/python/target.go | 17 +++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/gazelle/python/kinds.go b/gazelle/python/kinds.go index 83da53edbd..b370118dcd 100644 --- a/gazelle/python/kinds.go +++ b/gazelle/python/kinds.go @@ -60,10 +60,12 @@ var pyKinds = map[string]rule.KindInfo{ SubstituteAttrs: map[string]bool{}, MergeableAttrs: map[string]bool{ "srcs": true, + "deps_to_remove": true, }, ResolveAttrs: map[string]bool{ "deps": true, "pyi_deps": true, + "deps_to_remove": true, }, }, pyTestKind: { diff --git a/gazelle/python/resolve.go b/gazelle/python/resolve.go index 0dd80841d4..9d6deb4a3d 100644 --- a/gazelle/python/resolve.go +++ b/gazelle/python/resolve.go @@ -39,6 +39,8 @@ const ( // resolvedDepsKey is the attribute key used to pass dependencies that don't // need to be resolved by the dependency resolver in the Resolver step. resolvedDepsKey = "_gazelle_python_resolved_deps" + // depsToRemoveKey is the attribute key used to store the deps_to_remove list + depsToRemoveKey = "_gazelle_python_deps_to_remove" ) // Resolver satisfies the resolve.Resolver interface. It resolves dependencies @@ -356,6 +358,7 @@ func (py *Resolver) Resolve( if cfg.GeneratePyiDeps() { if !deps.Empty() { r.SetAttr("deps", convertDependencySetToExpr(deps)) + r.SetAttr("deps_to_remove", convertDependencySetToExpr(deps)) } if !pyiDeps.Empty() { r.SetAttr("pyi_deps", convertDependencySetToExpr(pyiDeps)) @@ -368,6 +371,7 @@ func (py *Resolver) Resolve( if !combinedDeps.Empty() { r.SetAttr("deps", convertDependencySetToExpr(combinedDeps)) + r.SetAttr("deps_to_remove", convertDependencySetToExpr(combinedDeps)) } } } diff --git a/gazelle/python/target.go b/gazelle/python/target.go index 3b7f464335..8729fdcf95 100644 --- a/gazelle/python/target.go +++ b/gazelle/python/target.go @@ -33,6 +33,7 @@ type targetBuilder struct { siblingSrcs *treeset.Set deps *treeset.Set resolvedDeps *treeset.Set + depsToRemove *treeset.Set visibility *treeset.Set main *string imports []string @@ -50,6 +51,7 @@ func newTargetBuilder(kind, name, pythonProjectRoot, bzlPackage string, siblingS siblingSrcs: siblingSrcs, deps: treeset.NewWith(moduleComparator), resolvedDeps: treeset.NewWith(godsutils.StringComparator), + depsToRemove: treeset.NewWith(godsutils.StringComparator), visibility: treeset.NewWith(godsutils.StringComparator), } } @@ -100,6 +102,20 @@ func (t *targetBuilder) addResolvedDependencies(deps []string) *targetBuilder { return t } +// addDepToRemove adds a single dependency to be removed to the target. +func (t *targetBuilder) addDepToRemove(dep string) *targetBuilder { + t.depsToRemove.Add(dep) + return t +} + +// addDepsToRemove adds multiple dependencies to be removed to the target. +func (t *targetBuilder) addDepsToRemove(deps []string) *targetBuilder { + for _, dep := range deps { + t.addDepToRemove(dep) + } + return t +} + // addVisibility adds visibility labels to the target. func (t *targetBuilder) addVisibility(visibility []string) *targetBuilder { for _, item := range visibility { @@ -161,5 +177,6 @@ func (t *targetBuilder) build() *rule.Rule { r.SetAttr("testonly", true) } r.SetPrivateAttr(resolvedDepsKey, t.resolvedDeps) + r.SetPrivateAttr(depsToRemoveKey, t.depsToRemove) return r } From 4b0d59426b9e49eac39c03ad44272fbaab598dbd Mon Sep 17 00:00:00 2001 From: Vivek Dasari Date: Wed, 16 Jul 2025 07:38:51 -0700 Subject: [PATCH 11/12] correctly only add deps_to_remove --- gazelle/python/language.go | 6 +- gazelle/python/resolve.go | 191 +++++++++++++++++- gazelle/python/target.go | 4 + .../BUILD.in | 1 + .../BUILD.out | 35 ++++ .../MODULE.bazel | 6 + .../README.md | 28 +++ .../WORKSPACE | 1 + .../__init__.py | 15 ++ .../application.py | 30 +++ .../deps-order.txt | 7 + .../foundation.py | 27 +++ .../middleware.py | 25 +++ .../test.yaml | 15 ++ .../deps_to_remove_with_order/BUILD.in | 1 + .../deps_to_remove_with_order/BUILD.out | 32 +++ .../deps_to_remove_with_order/README.md | 26 +++ .../deps_to_remove_with_order/WORKSPACE | 1 + .../deps_to_remove_with_order/__init__.py | 15 ++ .../deps_to_remove_with_order/core.py | 23 +++ .../deps_to_remove_with_order/deps-order.txt | 3 + .../deps_to_remove_with_order/high_level.py | 30 +++ .../deps_to_remove_with_order/test.yaml | 15 ++ .../deps_to_remove_with_order/utils.py | 27 +++ 24 files changed, 560 insertions(+), 4 deletions(-) create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/BUILD.in create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/BUILD.out create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/MODULE.bazel create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/README.md create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/WORKSPACE create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/__init__.py create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/application.py create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/deps-order.txt create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/foundation.py create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/middleware.py create mode 100644 gazelle/python/testdata/deps_to_remove_ordering_violation/test.yaml create mode 100644 gazelle/python/testdata/deps_to_remove_with_order/BUILD.in create mode 100644 gazelle/python/testdata/deps_to_remove_with_order/BUILD.out create mode 100644 gazelle/python/testdata/deps_to_remove_with_order/README.md create mode 100644 gazelle/python/testdata/deps_to_remove_with_order/WORKSPACE create mode 100644 gazelle/python/testdata/deps_to_remove_with_order/__init__.py create mode 100644 gazelle/python/testdata/deps_to_remove_with_order/core.py create mode 100644 gazelle/python/testdata/deps_to_remove_with_order/deps-order.txt create mode 100644 gazelle/python/testdata/deps_to_remove_with_order/high_level.py create mode 100644 gazelle/python/testdata/deps_to_remove_with_order/test.yaml create mode 100644 gazelle/python/testdata/deps_to_remove_with_order/utils.py diff --git a/gazelle/python/language.go b/gazelle/python/language.go index 56eb97b043..cab4fb2392 100644 --- a/gazelle/python/language.go +++ b/gazelle/python/language.go @@ -28,5 +28,9 @@ type Python struct { // NewLanguage initializes a new Python that satisfies the language.Language // interface. This is the entrypoint for the extension initialization. func NewLanguage() language.Language { - return &Python{} + return &Python{ + Resolver: Resolver{ + depsOrderResolver: NewDepsOrderResolver(), + }, + } } diff --git a/gazelle/python/resolve.go b/gazelle/python/resolve.go index 9d6deb4a3d..125e573827 100644 --- a/gazelle/python/resolve.go +++ b/gazelle/python/resolve.go @@ -15,6 +15,7 @@ package python import ( + "bufio" "fmt" "log" "os" @@ -41,11 +42,133 @@ const ( resolvedDepsKey = "_gazelle_python_resolved_deps" // depsToRemoveKey is the attribute key used to store the deps_to_remove list depsToRemoveKey = "_gazelle_python_deps_to_remove" + // srcsForOrderingKey is the attribute key used to store source files for ordering constraints + srcsForOrderingKey = "_gazelle_python_srcs_for_ordering" + // depsOrderFilename is the name of the file that contains the dependency order + depsOrderFilename = "deps-order.txt" ) +// DepsOrderResolver holds the dependency order information parsed from deps-order.txt +type DepsOrderResolver struct { + fileToIndex map[string]int + loaded bool + // importToSrcs maps import names to their source files (pkg-relative paths) + importToSrcs map[string][]string +} + +// NewDepsOrderResolver creates a new DepsOrderResolver +func NewDepsOrderResolver() *DepsOrderResolver { + return &DepsOrderResolver{ + fileToIndex: make(map[string]int), + loaded: false, + importToSrcs: make(map[string][]string), + } +} + +// LoadDepsOrder loads the deps-order.txt file from the repository root +func (d *DepsOrderResolver) LoadDepsOrder(repoRoot string) error { + if d.loaded { + return nil + } + + depsOrderPath := filepath.Join(repoRoot, depsOrderFilename) + file, err := os.Open(depsOrderPath) + if err != nil { + if os.IsNotExist(err) { + // File doesn't exist, which is fine - we just won't use deps ordering + d.loaded = true + return nil + } + return fmt.Errorf("failed to open %s: %v", depsOrderFilename, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + index := 0 + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue // Skip empty lines and comments + } + d.fileToIndex[line] = index + index++ + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("failed to read %s: %v", depsOrderFilename, err) + } + + d.loaded = true + return nil +} + +// GetAverageIndex calculates the average index for a set of source files +func (d *DepsOrderResolver) GetAverageIndex(srcs []string) float64 { + if len(d.fileToIndex) == 0 { + return 0 // No ordering file, return 0 + } + + totalIndex := 0 + validSrcs := 0 + for _, src := range srcs { + // Try both the full path and just the filename + filename := filepath.Base(src) + if index, exists := d.fileToIndex[src]; exists { + totalIndex += index + validSrcs++ + } else if index, exists := d.fileToIndex[filename]; exists { + totalIndex += index + validSrcs++ + } + } + + if validSrcs == 0 { + return float64(len(d.fileToIndex)) // Files not in order get max index + } + + return float64(totalIndex) / float64(validSrcs) +} + +// ShouldAddToDepsToRemove returns true if the dependency should be added to deps_to_remove based on ordering constraints +func (d *DepsOrderResolver) ShouldAddToDepsToRemove(currentTargetSrcs []string, depTargetSrcs []string) bool { + if len(d.fileToIndex) == 0 { + return false // No ordering constraints + } + + currentAvg := d.GetAverageIndex(currentTargetSrcs) + depAvg := d.GetAverageIndex(depTargetSrcs) + + // If current target has lower average index than dependency, the dependency should be removed + return currentAvg < depAvg +} + +// RegisterImportSources registers the mapping between import specs and their source files +func (d *DepsOrderResolver) RegisterImportSources(importSpecs []resolve.ImportSpec, pkgPath string, srcs []string) { + // Convert sources to repo-relative paths + repoRelativeSrcs := make([]string, 0, len(srcs)) + for _, src := range srcs { + repoRelativeSrcs = append(repoRelativeSrcs, filepath.Join(pkgPath, src)) + } + + // Register each import spec + for _, spec := range importSpecs { + d.importToSrcs[spec.Imp] = repoRelativeSrcs + } +} + +// getSourcesForImport gets the source files for a given import name using the registered mappings +func (d *DepsOrderResolver) getSourcesForImport(importName string) []string { + if srcs, ok := d.importToSrcs[importName]; ok { + return srcs + } + return []string{} +} + // Resolver satisfies the resolve.Resolver interface. It resolves dependencies // in rules generated by this extension. -type Resolver struct{} +type Resolver struct{ + depsOrderResolver *DepsOrderResolver +} // Name returns the name of the language. This is the prefix of the kinds of // rules generated. E.g. py_library and py_binary. @@ -78,6 +201,10 @@ func (py *Resolver) Imports(c *config.Config, r *rule.Rule, f *rule.File) []reso if len(provides) == 0 { return nil } + + // Register the import-to-source mappings for dependency ordering + py.depsOrderResolver.RegisterImportSources(provides, f.Pkg, srcs) + return provides } @@ -327,6 +454,22 @@ func (py *Resolver) Resolve( } matchLabel := filteredMatches[0].Label.Rel(from.Repo, from.Pkg) dep := matchLabel.String() + + // Register the mapping from dependency label to its source files + // This allows us to look up source files during deps_to_remove creation + match := filteredMatches[0] + depSrcsPaths := make([]string, 0) + // Try to infer source file from the import name + if strings.Contains(moduleName, ".") { + parts := strings.Split(moduleName, ".") + srcFile := parts[len(parts)-1] + ".py" + depSrcsPaths = append(depSrcsPaths, filepath.Join(match.Label.Pkg, srcFile)) + } else { + srcFile := moduleName + ".py" + depSrcsPaths = append(depSrcsPaths, filepath.Join(match.Label.Pkg, srcFile)) + } + py.depsOrderResolver.importToSrcs[dep] = depSrcsPaths + addDependency(dep, mod.TypeCheckingOnly, deps, pyiDeps) if explainDependency == dep { log.Printf("Explaining dependency (%s): "+ @@ -355,10 +498,49 @@ func (py *Resolver) Resolve( addResolvedDeps(r, deps) + // Load deps order constraints if available + err := py.depsOrderResolver.LoadDepsOrder(c.RepoRoot) + if err != nil { + log.Printf("Warning: failed to load deps-order.txt: %v", err) + } + + // Get current rule's sources for ordering comparison + currentSrcs := r.AttrStrings("srcs") + // Convert relative paths to paths relative to repo root + currentSrcsPaths := make([]string, 0, len(currentSrcs)) + for _, src := range currentSrcs { + currentSrcsPaths = append(currentSrcsPaths, filepath.Join(from.Pkg, src)) + } + + // Function to create deps_to_remove based on ordering constraints + createDepsToRemove := func(allDeps *treeset.Set) *treeset.Set { + depsToRemove := treeset.NewWith(godsutils.StringComparator) + + // If we have ordering constraints, check each dependency + if len(py.depsOrderResolver.fileToIndex) > 0 { + allDeps.Each(func(_ int, dep interface{}) { + depLabel := dep.(string) + + // Get the source files for this dependency using the registered mappings + depSrcs := py.depsOrderResolver.getSourcesForImport(depLabel) + + // Check if this dependency should be added to deps_to_remove based on ordering + if py.depsOrderResolver.ShouldAddToDepsToRemove(currentSrcsPaths, depSrcs) { + depsToRemove.Add(dep) + } + }) + } + + return depsToRemove + } + if cfg.GeneratePyiDeps() { if !deps.Empty() { r.SetAttr("deps", convertDependencySetToExpr(deps)) - r.SetAttr("deps_to_remove", convertDependencySetToExpr(deps)) + depsToRemove := createDepsToRemove(deps) + if !depsToRemove.Empty() { + r.SetAttr("deps_to_remove", convertDependencySetToExpr(depsToRemove)) + } } if !pyiDeps.Empty() { r.SetAttr("pyi_deps", convertDependencySetToExpr(pyiDeps)) @@ -371,7 +553,10 @@ func (py *Resolver) Resolve( if !combinedDeps.Empty() { r.SetAttr("deps", convertDependencySetToExpr(combinedDeps)) - r.SetAttr("deps_to_remove", convertDependencySetToExpr(combinedDeps)) + depsToRemove := createDepsToRemove(combinedDeps) + if !depsToRemove.Empty() { + r.SetAttr("deps_to_remove", convertDependencySetToExpr(depsToRemove)) + } } } } diff --git a/gazelle/python/target.go b/gazelle/python/target.go index 8729fdcf95..260debd1f2 100644 --- a/gazelle/python/target.go +++ b/gazelle/python/target.go @@ -178,5 +178,9 @@ func (t *targetBuilder) build() *rule.Rule { } r.SetPrivateAttr(resolvedDepsKey, t.resolvedDeps) r.SetPrivateAttr(depsToRemoveKey, t.depsToRemove) + // Store source files for ordering constraints + if !t.srcs.Empty() { + r.SetPrivateAttr(srcsForOrderingKey, t.srcs) + } return r } diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/BUILD.in b/gazelle/python/testdata/deps_to_remove_ordering_violation/BUILD.in new file mode 100644 index 0000000000..af2c2cea4b --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_generation_mode file diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/BUILD.out b/gazelle/python/testdata/deps_to_remove_ordering_violation/BUILD.out new file mode 100644 index 0000000000..44c5fe0b56 --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/BUILD.out @@ -0,0 +1,35 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode file + +py_library( + name = "__init__", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "application", + srcs = ["application.py"], + visibility = ["//:__subpackages__"], + deps = [ + ":foundation", + ":middleware", + ], +) + +py_library( + name = "foundation", + srcs = ["foundation.py"], + deps_to_remove = [":middleware"], + visibility = ["//:__subpackages__"], + deps = [":middleware"], +) + +py_library( + name = "middleware", + srcs = ["middleware.py"], + deps_to_remove = [":application"], + visibility = ["//:__subpackages__"], + deps = [":application"], +) diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/MODULE.bazel b/gazelle/python/testdata/deps_to_remove_ordering_violation/MODULE.bazel new file mode 100644 index 0000000000..00bb18361f --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/MODULE.bazel @@ -0,0 +1,6 @@ +############################################################################### +# Bazel now uses Bzlmod by default to manage external dependencies. +# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel. +# +# For more details, please check https://github.com/bazelbuild/bazel/issues/18958 +############################################################################### diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/README.md b/gazelle/python/testdata/deps_to_remove_ordering_violation/README.md new file mode 100644 index 0000000000..c2b752e48e --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/README.md @@ -0,0 +1,28 @@ +# Test case for deps_to_remove with ordering violations + +This test case verifies that the `deps_to_remove` attribute is correctly populated +when there are dependency ordering violations defined in `deps-order.txt`. + +## Test scenario: + +1. **deps-order.txt** defines the following order: + - `foundation.py` (index 0) - foundational code + - `middleware.py` (index 1) - middle layer + - `application.py` (index 2) - application layer + +2. **Dependencies**: + - `application.py` imports `middleware` and `foundation` (valid - higher depending on lower) + - `middleware.py` imports `application` (INVALID - lower depending on higher) + - `foundation.py` imports `middleware` (INVALID - lower depending on higher) + +3. **Expected behavior**: + - All dependencies should appear in `deps` + - Violating dependencies should also appear in `deps_to_remove`: + - `middleware`'s dependency on `application` should be in `deps_to_remove` + - `foundation`'s dependency on `middleware` should be in `deps_to_remove` + +## Files: +- `foundation.py` - foundational functionality (illegally depends on middleware) +- `middleware.py` - middle layer (illegally depends on application) +- `application.py` - application layer +- `deps-order.txt` - defines allowed dependency order diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/WORKSPACE b/gazelle/python/testdata/deps_to_remove_ordering_violation/WORKSPACE new file mode 100644 index 0000000000..e2025846c0 --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "deps_to_remove_ordering_violation_test") \ No newline at end of file diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/__init__.py b/gazelle/python/testdata/deps_to_remove_ordering_violation/__init__.py new file mode 100644 index 0000000000..3e68ab20ae --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Empty init file for test package \ No newline at end of file diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/application.py b/gazelle/python/testdata/deps_to_remove_ordering_violation/application.py new file mode 100644 index 0000000000..b47512f007 --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/application.py @@ -0,0 +1,30 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Application layer - depends on middleware and foundation (valid).""" + +import foundation +import middleware + +def app_function(): + """Application function using both foundation and middleware.""" + return f"app: {foundation.foundation_function()} + {middleware.middleware_function()}" + +def get_app_data(): + """Get application data.""" + return { + "app": True, + "foundation": foundation.get_foundation_data(), + "middleware": middleware.process_data() + } \ No newline at end of file diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/deps-order.txt b/gazelle/python/testdata/deps_to_remove_ordering_violation/deps-order.txt new file mode 100644 index 0000000000..4472cd3266 --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/deps-order.txt @@ -0,0 +1,7 @@ +# Dependency ordering constraints +# Files listed earlier can be depended upon by files listed later +# but not vice versa + +foundation.py +middleware.py +application.py \ No newline at end of file diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/foundation.py b/gazelle/python/testdata/deps_to_remove_ordering_violation/foundation.py new file mode 100644 index 0000000000..a66ba7e70b --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/foundation.py @@ -0,0 +1,27 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Foundation layer - ILLEGALLY depends on middleware (ordering violation).""" + +import middleware # This violates ordering constraints! + + +def foundation_function(): + """Foundation function that uses middleware.""" + return f"foundation: {middleware.middleware_function()}" + + +def get_foundation_data(): + """Get foundation data.""" + return {"foundation": True, "data": middleware.process_data()} diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/middleware.py b/gazelle/python/testdata/deps_to_remove_ordering_violation/middleware.py new file mode 100644 index 0000000000..13c5e2c3e9 --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/middleware.py @@ -0,0 +1,25 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Middleware layer - ILLEGALLY depends on application (ordering violation).""" + +import application # This violates ordering constraints! + +def middleware_function(): + """Middleware function that illegally uses application layer.""" + return f"middleware: {application.app_function()}" + +def process_data(): + """Process data using application layer (violation).""" + return application.get_app_data() \ No newline at end of file diff --git a/gazelle/python/testdata/deps_to_remove_ordering_violation/test.yaml b/gazelle/python/testdata/deps_to_remove_ordering_violation/test.yaml new file mode 100644 index 0000000000..fcea77710f --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_ordering_violation/test.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- diff --git a/gazelle/python/testdata/deps_to_remove_with_order/BUILD.in b/gazelle/python/testdata/deps_to_remove_with_order/BUILD.in new file mode 100644 index 0000000000..af2c2cea4b --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_with_order/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_generation_mode file diff --git a/gazelle/python/testdata/deps_to_remove_with_order/BUILD.out b/gazelle/python/testdata/deps_to_remove_with_order/BUILD.out new file mode 100644 index 0000000000..d90c9b35a4 --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_with_order/BUILD.out @@ -0,0 +1,32 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode file + +py_library( + name = "__init__", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "core", + srcs = ["core.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "high_level", + srcs = ["high_level.py"], + visibility = ["//:__subpackages__"], + deps = [ + ":core", + ":utils", + ], +) + +py_library( + name = "utils", + srcs = ["utils.py"], + visibility = ["//:__subpackages__"], + deps = [":core"], +) diff --git a/gazelle/python/testdata/deps_to_remove_with_order/README.md b/gazelle/python/testdata/deps_to_remove_with_order/README.md new file mode 100644 index 0000000000..2290f70d71 --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_with_order/README.md @@ -0,0 +1,26 @@ +# Test case for deps_to_remove with deps-order.txt + +This test case verifies that the `deps_to_remove` attribute is correctly populated +based on the dependency ordering constraints defined in `deps-order.txt`. + +## Test scenario: + +1. **deps-order.txt** defines the following order: + - `core.py` (index 0) - foundational code + - `utils.py` (index 1) - utility functions + - `high_level.py` (index 2) - high-level functionality + +2. **Dependencies**: + - `high_level.py` imports `core` and `utils` (valid - higher index depending on lower) + - `utils.py` imports `core` (valid - higher index depending on lower) + - `core.py` imports nothing (valid - no dependencies) + +3. **Expected behavior**: + - All dependencies should appear in `deps` + - No dependencies should appear in `deps_to_remove` (all are valid) + +## Files: +- `core.py` - basic functionality +- `utils.py` - depends on core +- `high_level.py` - depends on both core and utils +- `deps-order.txt` - defines allowed dependency order \ No newline at end of file diff --git a/gazelle/python/testdata/deps_to_remove_with_order/WORKSPACE b/gazelle/python/testdata/deps_to_remove_with_order/WORKSPACE new file mode 100644 index 0000000000..e9c57a5fb8 --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_with_order/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "deps_to_remove_with_order_test") \ No newline at end of file diff --git a/gazelle/python/testdata/deps_to_remove_with_order/__init__.py b/gazelle/python/testdata/deps_to_remove_with_order/__init__.py new file mode 100644 index 0000000000..3e68ab20ae --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_with_order/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Empty init file for test package \ No newline at end of file diff --git a/gazelle/python/testdata/deps_to_remove_with_order/core.py b/gazelle/python/testdata/deps_to_remove_with_order/core.py new file mode 100644 index 0000000000..3058784620 --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_with_order/core.py @@ -0,0 +1,23 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Core foundational functionality - no dependencies.""" + +def core_function(): + """Basic core function.""" + return "core functionality" + +def get_core_value(): + """Get a core value.""" + return 42 \ No newline at end of file diff --git a/gazelle/python/testdata/deps_to_remove_with_order/deps-order.txt b/gazelle/python/testdata/deps_to_remove_with_order/deps-order.txt new file mode 100644 index 0000000000..07b49b5245 --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_with_order/deps-order.txt @@ -0,0 +1,3 @@ +core.py +utils.py +high_level.py diff --git a/gazelle/python/testdata/deps_to_remove_with_order/high_level.py b/gazelle/python/testdata/deps_to_remove_with_order/high_level.py new file mode 100644 index 0000000000..b6ff8eca5e --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_with_order/high_level.py @@ -0,0 +1,30 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""High-level functionality - depends on both core and utils.""" + +import core +import utils + +def high_level_operation(): + """High-level operation using both core and utils.""" + core_result = core.core_function() + utils_result = utils.utility_function() + return f"High level: {core_result} + {utils_result}" + +def process_data(): + """Process data using all available functionality.""" + value = core.get_core_value() + formatted = utils.format_output(value) + return f"Processed: {formatted}" \ No newline at end of file diff --git a/gazelle/python/testdata/deps_to_remove_with_order/test.yaml b/gazelle/python/testdata/deps_to_remove_with_order/test.yaml new file mode 100644 index 0000000000..fcea77710f --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_with_order/test.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- diff --git a/gazelle/python/testdata/deps_to_remove_with_order/utils.py b/gazelle/python/testdata/deps_to_remove_with_order/utils.py new file mode 100644 index 0000000000..360c2bb05e --- /dev/null +++ b/gazelle/python/testdata/deps_to_remove_with_order/utils.py @@ -0,0 +1,27 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility functions - depends on core.""" + +import core + +def utility_function(): + """Utility function that uses core functionality.""" + base_value = core.get_core_value() + return f"utility result: {base_value * 2}" + +def format_output(value): + """Format output using core function.""" + core_result = core.core_function() + return f"{core_result} -> {value}" \ No newline at end of file From 46377ad2cb474846b8fc49573a8b738c98320d34 Mon Sep 17 00:00:00 2001 From: Vivek Dasari Date: Wed, 16 Jul 2025 10:16:35 -0700 Subject: [PATCH 12/12] use median instead of average --- SUMMARY.md | 188 ++++++++++++++++++++++++++++++++++++++ gazelle/python/resolve.go | 36 +++++--- 2 files changed, 210 insertions(+), 14 deletions(-) create mode 100644 SUMMARY.md diff --git a/SUMMARY.md b/SUMMARY.md new file mode 100644 index 0000000000..f9eb9d5263 --- /dev/null +++ b/SUMMARY.md @@ -0,0 +1,188 @@ +# Implementation Summary: `deps_to_remove` Feature for Gazelle Python Extension + +## Overview + +This document summarizes the implementation of the `deps_to_remove` feature for `py_library` targets in the Gazelle Python extension. The feature allows automatic generation of a `deps_to_remove` attribute that contains dependencies violating ordering constraints defined in a `deps-order.txt` file. + +## Feature Requirements + +- **All dependencies** must be included in the `deps` attribute (normal behavior) +- **Violating dependencies** must also be included in the `deps_to_remove` attribute +- **Dependency ordering** is defined by a `deps-order.txt` file at the repository root +- **Files listed earlier** in `deps-order.txt` can be depended upon by files listed later, but not vice versa + +## Implementation Details + +### 1. Attribute Support (`kinds.go`) + +**File**: `/workspaces/rules_python/gazelle/python/kinds.go` + +Added `deps_to_remove` attribute support to `pyLibraryKind`: +```go +MergeableAttrs: map[string]bool{ + "srcs": true, + "deps_to_remove": true, +}, +ResolveAttrs: map[string]bool{ + "deps": true, + "pyi_deps": true, + "deps_to_remove": true, +}, +``` + +### 2. Target Builder Enhancement (`target.go`) + +**File**: `/workspaces/rules_python/gazelle/python/target.go` + +- Added `depsToRemove` field to `targetBuilder` struct +- Added helper methods: `addDepToRemove()`, `addDepsToRemove()` +- Updated `build()` method to store source files for ordering constraints + +### 3. Dependency Order Resolution (`resolve.go`) + +**File**: `/workspaces/rules_python/gazelle/python/resolve.go` + +#### Core Components: + +**DepsOrderResolver Structure:** +```go +type DepsOrderResolver struct { + fileToIndex map[string]int // File to ordering index mapping + loaded bool // Loading state + importToSrcs map[string][]string // Import to source files mapping +} +``` + +**Key Methods:** +- `LoadDepsOrder()` - Parses `deps-order.txt` file +- `GetMedianIndex()` - Calculates median ordering index for source files +- `ShouldAddToDepsToRemove()` - Determines if dependency violates ordering +- `RegisterImportSources()` - Maps import specs to source files + +#### Dependency Processing Logic: + +1. **During Import Registration**: Map import specs to their source files +2. **During Dependency Resolution**: Register dependency labels to source files +3. **During Rule Finalization**: Apply ordering constraints to create `deps_to_remove` + +### 4. Language Integration (`language.go`) + +**File**: `/workspaces/rules_python/gazelle/python/language.go` + +Updated `NewLanguage()` to initialize the resolver with `DepsOrderResolver`: +```go +return &Python{ + Resolver: Resolver{ + depsOrderResolver: NewDepsOrderResolver(), + }, +} +``` + +## Algorithm Details + +### Ordering Constraint Logic + +1. **File Indexing**: Each file in `deps-order.txt` gets an index (0, 1, 2, ...) +2. **Median Calculation**: For targets with multiple sources, calculate median index +3. **Violation Detection**: If `currentTargetIndex < dependencyTargetIndex`, it's a violation +4. **Attribute Population**: Violating dependencies are added to both `deps` and `deps_to_remove` + +### Path Matching Strategy + +The implementation handles path matching flexibly: +- Tries exact path matches first (`pkg/file.py`) +- Falls back to filename matches (`file.py`) +- Supports both repo-relative and package-relative paths + +## Test Coverage + +### Test Case 1: Valid Dependencies (`deps_to_remove_with_order`) + +**Files**: `core.py` → `utils.py` → `high_level.py` + +**Scenario**: All dependencies follow correct ordering +- `utils.py` depends on `core.py` ✅ (valid: index 1 → index 0) +- `high_level.py` depends on both ✅ (valid: index 2 → index 0,1) + +**Expected Result**: No `deps_to_remove` attributes (all dependencies are valid) + +### Test Case 2: Ordering Violations (`deps_to_remove_ordering_violation`) + +**Files**: `foundation.py` → `middleware.py` → `application.py` + +**Scenario**: Contains dependency ordering violations +- `foundation.py` depends on `middleware.py` ❌ (violation: index 0 → index 1) +- `middleware.py` depends on `application.py` ❌ (violation: index 1 → index 2) + +**Expected Result**: +- `foundation` target: `deps_to_remove = [":middleware"]` +- `middleware` target: `deps_to_remove = [":application"]` + +## File Structure + +``` +gazelle/python/ +├── kinds.go # Attribute definitions +├── target.go # Target building logic +├── resolve.go # Dependency resolution & ordering +├── language.go # Language initialization +└── testdata/ + ├── deps_to_remove_with_order/ # Valid dependencies test + │ ├── deps-order.txt + │ ├── core.py, utils.py, high_level.py + │ ├── BUILD.in, BUILD.out + │ └── test.yaml + └── deps_to_remove_ordering_violation/ # Violation test + ├── deps-order.txt + ├── foundation.py, middleware.py, application.py + ├── BUILD.in, BUILD.out + └── test.yaml +``` + +## Usage + +### 1. Create `deps-order.txt` at repository root: +``` +# Comments are supported +core/base.py +utils/helpers.py +features/advanced.py +``` + +### 2. Run Gazelle: +```bash +bazel run //:gazelle +``` + +### 3. Generated BUILD files will include `deps_to_remove`: +```python +py_library( + name = "advanced", + srcs = ["advanced.py"], + deps = [ + "//core:base", # Valid dependency + "//utils:helpers", # Valid dependency + ], + # deps_to_remove is empty - no violations +) + +py_library( + name = "helpers", + srcs = ["helpers.py"], + deps = ["//features:advanced"], # Invalid dependency + deps_to_remove = ["//features:advanced"], # Marked for removal +) +``` + +## Benefits + +1. **Automated Detection**: Automatically identifies dependency ordering violations +2. **Build Compatibility**: All dependencies remain in `deps` for build correctness +3. **Tooling Integration**: `deps_to_remove` can be used by linters, analyzers, etc. +4. **Flexible Configuration**: Simple text file configuration +5. **Backward Compatible**: No `deps-order.txt` means no constraints applied + + +--- + +This implementation provides a robust foundation for dependency ordering enforcement in Python projects using Bazel and Gazelle. diff --git a/gazelle/python/resolve.go b/gazelle/python/resolve.go index 125e573827..957e108391 100644 --- a/gazelle/python/resolve.go +++ b/gazelle/python/resolve.go @@ -20,6 +20,7 @@ import ( "log" "os" "path/filepath" + "sort" "strings" "github.com/bazelbuild/bazel-gazelle/config" @@ -102,31 +103,38 @@ func (d *DepsOrderResolver) LoadDepsOrder(repoRoot string) error { return nil } -// GetAverageIndex calculates the average index for a set of source files -func (d *DepsOrderResolver) GetAverageIndex(srcs []string) float64 { +// GetMedianIndex calculates the median index for a set of source files +func (d *DepsOrderResolver) GetMedianIndex(srcs []string) float64 { if len(d.fileToIndex) == 0 { return 0 // No ordering file, return 0 } - totalIndex := 0 - validSrcs := 0 + var indices []int for _, src := range srcs { // Try both the full path and just the filename filename := filepath.Base(src) if index, exists := d.fileToIndex[src]; exists { - totalIndex += index - validSrcs++ + indices = append(indices, index) } else if index, exists := d.fileToIndex[filename]; exists { - totalIndex += index - validSrcs++ + indices = append(indices, index) } } - if validSrcs == 0 { + if len(indices) == 0 { return float64(len(d.fileToIndex)) // Files not in order get max index } - return float64(totalIndex) / float64(validSrcs) + // Sort indices to find median + sort.Ints(indices) + n := len(indices) + + if n%2 == 0 { + // Even number of elements: average of two middle elements + return float64(indices[n/2-1]+indices[n/2]) / 2.0 + } else { + // Odd number of elements: middle element + return float64(indices[n/2]) + } } // ShouldAddToDepsToRemove returns true if the dependency should be added to deps_to_remove based on ordering constraints @@ -135,11 +143,11 @@ func (d *DepsOrderResolver) ShouldAddToDepsToRemove(currentTargetSrcs []string, return false // No ordering constraints } - currentAvg := d.GetAverageIndex(currentTargetSrcs) - depAvg := d.GetAverageIndex(depTargetSrcs) + currentMedian := d.GetMedianIndex(currentTargetSrcs) + depMedian := d.GetMedianIndex(depTargetSrcs) - // If current target has lower average index than dependency, the dependency should be removed - return currentAvg < depAvg + // If current target has lower median index than dependency, the dependency should be removed + return currentMedian < depMedian } // RegisterImportSources registers the mapping between import specs and their source files